1 /*
2 * Copyright (c) 2017-2020, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <arch_helpers.h>
8 #include <assert.h>
9 #include <stddef.h>
10 #include <string.h>
11
12 #include <bl31/bl31.h>
13 #include <bl31/ehf.h>
14 #include <bl31/interrupt_mgmt.h>
15 #include <common/bl_common.h>
16 #include <common/debug.h>
17 #include <common/runtime_svc.h>
18 #include <context.h>
19 #include <lib/cassert.h>
20 #include <lib/el3_runtime/pubsub.h>
21 #include <lib/utils.h>
22 #include <plat/common/platform.h>
23 #include <services/sdei.h>
24
25 #include "sdei_private.h"
26
27 #define MAJOR_VERSION 1ULL
28 #define MINOR_VERSION 0ULL
29 #define VENDOR_VERSION 0ULL
30
31 #define MAKE_SDEI_VERSION(_major, _minor, _vendor) \
32 ((((_major)) << 48ULL) | (((_minor)) << 32ULL) | (_vendor))
33
34 #define LOWEST_INTR_PRIORITY 0xff
35
36 #define is_valid_affinity(_mpidr) (plat_core_pos_by_mpidr(_mpidr) >= 0)
37
38 CASSERT(PLAT_SDEI_CRITICAL_PRI < PLAT_SDEI_NORMAL_PRI,
39 sdei_critical_must_have_higher_priority);
40
41 static unsigned int num_dyn_priv_slots, num_dyn_shrd_slots;
42
43 /* Initialise SDEI map entries */
init_map(sdei_ev_map_t * map)44 static void init_map(sdei_ev_map_t *map)
45 {
46 map->reg_count = 0;
47 }
48
49 /* Convert mapping to SDEI class */
map_to_class(sdei_ev_map_t * map)50 static sdei_class_t map_to_class(sdei_ev_map_t *map)
51 {
52 return is_event_critical(map) ? SDEI_CRITICAL : SDEI_NORMAL;
53 }
54
55 /* Clear SDEI event entries except state */
clear_event_entries(sdei_entry_t * se)56 static void clear_event_entries(sdei_entry_t *se)
57 {
58 se->ep = 0;
59 se->arg = 0;
60 se->affinity = 0;
61 se->reg_flags = 0;
62 }
63
64 /* Perform CPU-specific state initialisation */
sdei_cpu_on_init(const void * arg)65 static void *sdei_cpu_on_init(const void *arg)
66 {
67 unsigned int i;
68 sdei_ev_map_t *map;
69 sdei_entry_t *se;
70
71 /* Initialize private mappings on this CPU */
72 for_each_private_map(i, map) {
73 se = get_event_entry(map);
74 clear_event_entries(se);
75 se->state = 0;
76 }
77
78 SDEI_LOG("Private events initialized on %lx\n", read_mpidr_el1());
79
80 /* All PEs start with SDEI events masked */
81 (void) sdei_pe_mask();
82
83 return NULL;
84 }
85
86 /* CPU initialisation after wakeup from suspend */
sdei_cpu_wakeup_init(const void * arg)87 static void *sdei_cpu_wakeup_init(const void *arg)
88 {
89 SDEI_LOG("Events masked on %lx\n", read_mpidr_el1());
90
91 /* All PEs wake up with SDEI events masked */
92 sdei_pe_mask();
93
94 return 0;
95 }
96
97 /* Initialise an SDEI class */
sdei_class_init(sdei_class_t class)98 static void sdei_class_init(sdei_class_t class)
99 {
100 unsigned int i;
101 bool zero_found __unused = false;
102 int ev_num_so_far __unused;
103 sdei_ev_map_t *map;
104
105 /* Sanity check and configuration of shared events */
106 ev_num_so_far = -1;
107 for_each_shared_map(i, map) {
108 #if ENABLE_ASSERTIONS
109 /* Ensure mappings are sorted */
110 assert((ev_num_so_far < 0) || (map->ev_num > ev_num_so_far));
111
112 ev_num_so_far = map->ev_num;
113
114 /* Event 0 must not be shared */
115 assert(map->ev_num != SDEI_EVENT_0);
116
117 /* Check for valid event */
118 assert(map->ev_num >= 0);
119
120 /* Make sure it's a shared event */
121 assert(is_event_shared(map));
122
123 /* No shared mapping should have signalable property */
124 assert(!is_event_signalable(map));
125
126 /* Shared mappings can't be explicit */
127 assert(!is_map_explicit(map));
128 #endif
129
130 /* Skip initializing the wrong priority */
131 if (map_to_class(map) != class)
132 continue;
133
134 /* Platform events are always bound, so set the bound flag */
135 if (is_map_dynamic(map)) {
136 assert(map->intr == SDEI_DYN_IRQ);
137 assert(is_event_normal(map));
138 num_dyn_shrd_slots++;
139 } else {
140 /* Shared mappings must be bound to shared interrupt */
141 assert(plat_ic_is_spi(map->intr) != 0);
142 set_map_bound(map);
143 }
144
145 init_map(map);
146 }
147
148 /* Sanity check and configuration of private events for this CPU */
149 ev_num_so_far = -1;
150 for_each_private_map(i, map) {
151 #if ENABLE_ASSERTIONS
152 /* Ensure mappings are sorted */
153 assert((ev_num_so_far < 0) || (map->ev_num > ev_num_so_far));
154
155 ev_num_so_far = map->ev_num;
156
157 if (map->ev_num == SDEI_EVENT_0) {
158 zero_found = true;
159
160 /* Event 0 must be a Secure SGI */
161 assert(is_secure_sgi(map->intr));
162
163 /*
164 * Event 0 can have only have signalable flag (apart
165 * from being private
166 */
167 assert(map->map_flags == (SDEI_MAPF_SIGNALABLE |
168 SDEI_MAPF_PRIVATE));
169 } else {
170 /* No other mapping should have signalable property */
171 assert(!is_event_signalable(map));
172 }
173
174 /* Check for valid event */
175 assert(map->ev_num >= 0);
176
177 /* Make sure it's a private event */
178 assert(is_event_private(map));
179
180 /*
181 * Other than priority, explicit events can only have explicit
182 * and private flags set.
183 */
184 if (is_map_explicit(map)) {
185 assert((map->map_flags | SDEI_MAPF_CRITICAL) ==
186 (SDEI_MAPF_EXPLICIT | SDEI_MAPF_PRIVATE
187 | SDEI_MAPF_CRITICAL));
188 }
189 #endif
190
191 /* Skip initializing the wrong priority */
192 if (map_to_class(map) != class)
193 continue;
194
195 /* Platform events are always bound, so set the bound flag */
196 if (map->ev_num != SDEI_EVENT_0) {
197 if (is_map_dynamic(map)) {
198 assert(map->intr == SDEI_DYN_IRQ);
199 assert(is_event_normal(map));
200 num_dyn_priv_slots++;
201 } else if (is_map_explicit(map)) {
202 /*
203 * Explicit mappings don't have a backing
204 * SDEI interrupt, but verify that anyway.
205 */
206 assert(map->intr == SDEI_DYN_IRQ);
207 } else {
208 /*
209 * Private mappings must be bound to private
210 * interrupt.
211 */
212 assert(plat_ic_is_ppi((unsigned) map->intr) != 0);
213 set_map_bound(map);
214 }
215 }
216
217 init_map(map);
218 }
219
220 /* Ensure event 0 is in the mapping */
221 assert(zero_found);
222
223 (void) sdei_cpu_on_init(NULL);
224 }
225
226 /* SDEI dispatcher initialisation */
sdei_init(void)227 void sdei_init(void)
228 {
229 plat_sdei_setup();
230 sdei_class_init(SDEI_CRITICAL);
231 sdei_class_init(SDEI_NORMAL);
232
233 /* Register priority level handlers */
234 ehf_register_priority_handler(PLAT_SDEI_CRITICAL_PRI,
235 sdei_intr_handler);
236 ehf_register_priority_handler(PLAT_SDEI_NORMAL_PRI,
237 sdei_intr_handler);
238 }
239
240 /* Populate SDEI event entry */
set_sdei_entry(sdei_entry_t * se,uint64_t ep,uint64_t arg,unsigned int flags,uint64_t affinity)241 static void set_sdei_entry(sdei_entry_t *se, uint64_t ep, uint64_t arg,
242 unsigned int flags, uint64_t affinity)
243 {
244 assert(se != NULL);
245
246 se->ep = ep;
247 se->arg = arg;
248 se->affinity = (affinity & MPIDR_AFFINITY_MASK);
249 se->reg_flags = flags;
250 }
251
sdei_version(void)252 static uint64_t sdei_version(void)
253 {
254 return MAKE_SDEI_VERSION(MAJOR_VERSION, MINOR_VERSION, VENDOR_VERSION);
255 }
256
257 /* Validate flags and MPIDR values for REGISTER and ROUTING_SET calls */
validate_flags(uint64_t flags,uint64_t mpidr)258 static int validate_flags(uint64_t flags, uint64_t mpidr)
259 {
260 /* Validate flags */
261 switch (flags) {
262 case SDEI_REGF_RM_PE:
263 if (!is_valid_affinity(mpidr))
264 return SDEI_EINVAL;
265 break;
266 case SDEI_REGF_RM_ANY:
267 break;
268 default:
269 /* Unknown flags */
270 return SDEI_EINVAL;
271 }
272
273 return 0;
274 }
275
276 /* Set routing of an SDEI event */
sdei_event_routing_set(int ev_num,uint64_t flags,uint64_t mpidr)277 static int sdei_event_routing_set(int ev_num, uint64_t flags, uint64_t mpidr)
278 {
279 int ret;
280 unsigned int routing;
281 sdei_ev_map_t *map;
282 sdei_entry_t *se;
283
284 ret = validate_flags(flags, mpidr);
285 if (ret != 0)
286 return ret;
287
288 /* Check if valid event number */
289 map = find_event_map(ev_num);
290 if (map == NULL)
291 return SDEI_EINVAL;
292
293 /* The event must not be private */
294 if (is_event_private(map))
295 return SDEI_EINVAL;
296
297 se = get_event_entry(map);
298
299 sdei_map_lock(map);
300
301 if (!is_map_bound(map) || is_event_private(map)) {
302 ret = SDEI_EINVAL;
303 goto finish;
304 }
305
306 if (!can_sdei_state_trans(se, DO_ROUTING)) {
307 ret = SDEI_EDENY;
308 goto finish;
309 }
310
311 /* Choose appropriate routing */
312 routing = (unsigned int) ((flags == SDEI_REGF_RM_ANY) ?
313 INTR_ROUTING_MODE_ANY : INTR_ROUTING_MODE_PE);
314
315 /* Update event registration flag */
316 se->reg_flags = (unsigned int) flags;
317 if (flags == SDEI_REGF_RM_PE) {
318 se->affinity = (mpidr & MPIDR_AFFINITY_MASK);
319 }
320
321 /*
322 * ROUTING_SET is permissible only when event composite state is
323 * 'registered, disabled, and not running'. This means that the
324 * interrupt is currently disabled, and not active.
325 */
326 plat_ic_set_spi_routing(map->intr, routing, (u_register_t) mpidr);
327
328 finish:
329 sdei_map_unlock(map);
330
331 return ret;
332 }
333
334 /* Register handler and argument for an SDEI event */
sdei_event_register(int ev_num,uint64_t ep,uint64_t arg,uint64_t flags,uint64_t mpidr)335 static int64_t sdei_event_register(int ev_num,
336 uint64_t ep,
337 uint64_t arg,
338 uint64_t flags,
339 uint64_t mpidr)
340 {
341 int ret;
342 unsigned int routing;
343 sdei_entry_t *se;
344 sdei_ev_map_t *map;
345 sdei_state_t backup_state;
346
347 if ((ep == 0U) || (plat_sdei_validate_entry_point(
348 ep, sdei_client_el()) != 0)) {
349 return SDEI_EINVAL;
350 }
351
352 ret = validate_flags(flags, mpidr);
353 if (ret != 0)
354 return ret;
355
356 /* Check if valid event number */
357 map = find_event_map(ev_num);
358 if (map == NULL)
359 return SDEI_EINVAL;
360
361 /* Private events always target the PE */
362 if (is_event_private(map))
363 flags = SDEI_REGF_RM_PE;
364
365 se = get_event_entry(map);
366
367 /*
368 * Even though register operation is per-event (additionally for private
369 * events, registration is required individually), it has to be
370 * serialised with respect to bind/release, which are global operations.
371 * So we hold the lock throughout, unconditionally.
372 */
373 sdei_map_lock(map);
374
375 backup_state = se->state;
376 if (!can_sdei_state_trans(se, DO_REGISTER))
377 goto fallback;
378
379 /*
380 * When registering for dynamic events, make sure it's been bound
381 * already. This has to be the case as, without binding, the client
382 * can't know about the event number to register for.
383 */
384 if (is_map_dynamic(map) && !is_map_bound(map))
385 goto fallback;
386
387 if (is_event_private(map)) {
388 /* Multiple calls to register are possible for private events */
389 assert(map->reg_count >= 0);
390 } else {
391 /* Only single call to register is possible for shared events */
392 assert(map->reg_count == 0);
393 }
394
395 if (is_map_bound(map)) {
396 /* Meanwhile, did any PE ACK the interrupt? */
397 if (plat_ic_get_interrupt_active(map->intr) != 0U)
398 goto fallback;
399
400 /* The interrupt must currently owned by Non-secure */
401 if (plat_ic_get_interrupt_type(map->intr) != INTR_TYPE_NS)
402 goto fallback;
403
404 /*
405 * Disable forwarding of new interrupt triggers to CPU
406 * interface.
407 */
408 plat_ic_disable_interrupt(map->intr);
409
410 /*
411 * Any events that are triggered after register and before
412 * enable should remain pending. Clear any previous interrupt
413 * triggers which are pending (except for SGIs). This has no
414 * affect on level-triggered interrupts.
415 */
416 if (ev_num != SDEI_EVENT_0)
417 plat_ic_clear_interrupt_pending(map->intr);
418
419 /* Map interrupt to EL3 and program the correct priority */
420 plat_ic_set_interrupt_type(map->intr, INTR_TYPE_EL3);
421
422 /* Program the appropriate interrupt priority */
423 plat_ic_set_interrupt_priority(map->intr, sdei_event_priority(map));
424
425 /*
426 * Set the routing mode for shared event as requested. We
427 * already ensure that shared events get bound to SPIs.
428 */
429 if (is_event_shared(map)) {
430 routing = (unsigned int) ((flags == SDEI_REGF_RM_ANY) ?
431 INTR_ROUTING_MODE_ANY : INTR_ROUTING_MODE_PE);
432 plat_ic_set_spi_routing(map->intr, routing,
433 (u_register_t) mpidr);
434 }
435 }
436
437 /* Populate event entries */
438 set_sdei_entry(se, ep, arg, (unsigned int) flags, mpidr);
439
440 /* Increment register count */
441 map->reg_count++;
442
443 sdei_map_unlock(map);
444
445 return 0;
446
447 fallback:
448 /* Reinstate previous state */
449 se->state = backup_state;
450
451 sdei_map_unlock(map);
452
453 return SDEI_EDENY;
454 }
455
456 /* Enable SDEI event */
sdei_event_enable(int ev_num)457 static int64_t sdei_event_enable(int ev_num)
458 {
459 sdei_ev_map_t *map;
460 sdei_entry_t *se;
461 int ret;
462 bool before, after;
463
464 /* Check if valid event number */
465 map = find_event_map(ev_num);
466 if (map == NULL)
467 return SDEI_EINVAL;
468
469 se = get_event_entry(map);
470 ret = SDEI_EDENY;
471
472 if (is_event_shared(map))
473 sdei_map_lock(map);
474
475 before = GET_EV_STATE(se, ENABLED);
476 if (!can_sdei_state_trans(se, DO_ENABLE))
477 goto finish;
478 after = GET_EV_STATE(se, ENABLED);
479
480 /*
481 * Enable interrupt for bound events only if there's a change in enabled
482 * state.
483 */
484 if (is_map_bound(map) && (!before && after))
485 plat_ic_enable_interrupt(map->intr);
486
487 ret = 0;
488
489 finish:
490 if (is_event_shared(map))
491 sdei_map_unlock(map);
492
493 return ret;
494 }
495
496 /* Disable SDEI event */
sdei_event_disable(int ev_num)497 static int sdei_event_disable(int ev_num)
498 {
499 sdei_ev_map_t *map;
500 sdei_entry_t *se;
501 int ret;
502 bool before, after;
503
504 /* Check if valid event number */
505 map = find_event_map(ev_num);
506 if (map == NULL)
507 return SDEI_EINVAL;
508
509 se = get_event_entry(map);
510 ret = SDEI_EDENY;
511
512 if (is_event_shared(map))
513 sdei_map_lock(map);
514
515 before = GET_EV_STATE(se, ENABLED);
516 if (!can_sdei_state_trans(se, DO_DISABLE))
517 goto finish;
518 after = GET_EV_STATE(se, ENABLED);
519
520 /*
521 * Disable interrupt for bound events only if there's a change in
522 * enabled state.
523 */
524 if (is_map_bound(map) && (before && !after))
525 plat_ic_disable_interrupt(map->intr);
526
527 ret = 0;
528
529 finish:
530 if (is_event_shared(map))
531 sdei_map_unlock(map);
532
533 return ret;
534 }
535
536 /* Query SDEI event information */
sdei_event_get_info(int ev_num,int info)537 static int64_t sdei_event_get_info(int ev_num, int info)
538 {
539 sdei_entry_t *se;
540 sdei_ev_map_t *map;
541
542 uint64_t flags;
543 bool registered;
544 uint64_t affinity;
545
546 /* Check if valid event number */
547 map = find_event_map(ev_num);
548 if (map == NULL)
549 return SDEI_EINVAL;
550
551 se = get_event_entry(map);
552
553 if (is_event_shared(map))
554 sdei_map_lock(map);
555
556 /* Sample state under lock */
557 registered = GET_EV_STATE(se, REGISTERED);
558 flags = se->reg_flags;
559 affinity = se->affinity;
560
561 if (is_event_shared(map))
562 sdei_map_unlock(map);
563
564 switch (info) {
565 case SDEI_INFO_EV_TYPE:
566 return is_event_shared(map);
567
568 case SDEI_INFO_EV_NOT_SIGNALED:
569 return !is_event_signalable(map);
570
571 case SDEI_INFO_EV_PRIORITY:
572 return is_event_critical(map);
573
574 case SDEI_INFO_EV_ROUTING_MODE:
575 if (!is_event_shared(map))
576 return SDEI_EINVAL;
577 if (!registered)
578 return SDEI_EDENY;
579 return (flags == SDEI_REGF_RM_PE);
580
581 case SDEI_INFO_EV_ROUTING_AFF:
582 if (!is_event_shared(map))
583 return SDEI_EINVAL;
584 if (!registered)
585 return SDEI_EDENY;
586 if (flags != SDEI_REGF_RM_PE)
587 return SDEI_EINVAL;
588 return affinity;
589
590 default:
591 return SDEI_EINVAL;
592 }
593 }
594
595 /* Unregister an SDEI event */
sdei_event_unregister(int ev_num)596 static int sdei_event_unregister(int ev_num)
597 {
598 int ret = 0;
599 sdei_entry_t *se;
600 sdei_ev_map_t *map;
601
602 /* Check if valid event number */
603 map = find_event_map(ev_num);
604 if (map == NULL)
605 return SDEI_EINVAL;
606
607 se = get_event_entry(map);
608
609 /*
610 * Even though unregister operation is per-event (additionally for
611 * private events, unregistration is required individually), it has to
612 * be serialised with respect to bind/release, which are global
613 * operations. So we hold the lock throughout, unconditionally.
614 */
615 sdei_map_lock(map);
616
617 if (!can_sdei_state_trans(se, DO_UNREGISTER)) {
618 /*
619 * Even if the call is invalid, and the handler is running (for
620 * example, having unregistered from a running handler earlier),
621 * return pending error code; otherwise, return deny.
622 */
623 ret = GET_EV_STATE(se, RUNNING) ? SDEI_EPEND : SDEI_EDENY;
624
625 goto finish;
626 }
627
628 map->reg_count--;
629 if (is_event_private(map)) {
630 /* Multiple calls to register are possible for private events */
631 assert(map->reg_count >= 0);
632 } else {
633 /* Only single call to register is possible for shared events */
634 assert(map->reg_count == 0);
635 }
636
637 if (is_map_bound(map)) {
638 plat_ic_disable_interrupt(map->intr);
639
640 /*
641 * Clear pending interrupt. Skip for SGIs as they may not be
642 * cleared on interrupt controllers.
643 */
644 if (ev_num != SDEI_EVENT_0)
645 plat_ic_clear_interrupt_pending(map->intr);
646
647 assert(plat_ic_get_interrupt_type(map->intr) == INTR_TYPE_EL3);
648 plat_ic_set_interrupt_type(map->intr, INTR_TYPE_NS);
649 plat_ic_set_interrupt_priority(map->intr, LOWEST_INTR_PRIORITY);
650 }
651
652 clear_event_entries(se);
653
654 /*
655 * If the handler is running at the time of unregister, return the
656 * pending error code.
657 */
658 if (GET_EV_STATE(se, RUNNING))
659 ret = SDEI_EPEND;
660
661 finish:
662 sdei_map_unlock(map);
663
664 return ret;
665 }
666
667 /* Query status of an SDEI event */
sdei_event_status(int ev_num)668 static int sdei_event_status(int ev_num)
669 {
670 sdei_ev_map_t *map;
671 sdei_entry_t *se;
672 sdei_state_t state;
673
674 /* Check if valid event number */
675 map = find_event_map(ev_num);
676 if (map == NULL)
677 return SDEI_EINVAL;
678
679 se = get_event_entry(map);
680
681 if (is_event_shared(map))
682 sdei_map_lock(map);
683
684 /* State value directly maps to the expected return format */
685 state = se->state;
686
687 if (is_event_shared(map))
688 sdei_map_unlock(map);
689
690 return (int) state;
691 }
692
693 /* Bind an SDEI event to an interrupt */
sdei_interrupt_bind(unsigned int intr_num)694 static int sdei_interrupt_bind(unsigned int intr_num)
695 {
696 sdei_ev_map_t *map;
697 bool retry = true, shared_mapping;
698
699 /* SGIs are not allowed to be bound */
700 if (plat_ic_is_sgi(intr_num) != 0)
701 return SDEI_EINVAL;
702
703 shared_mapping = (plat_ic_is_spi(intr_num) != 0);
704 do {
705 /*
706 * Bail out if there is already an event for this interrupt,
707 * either platform-defined or dynamic.
708 */
709 map = find_event_map_by_intr(intr_num, shared_mapping);
710 if (map != NULL) {
711 if (is_map_dynamic(map)) {
712 if (is_map_bound(map)) {
713 /*
714 * Dynamic event, already bound. Return
715 * event number.
716 */
717 return map->ev_num;
718 }
719 } else {
720 /* Binding non-dynamic event */
721 return SDEI_EINVAL;
722 }
723 }
724
725 /*
726 * The interrupt is not bound yet. Try to find a free slot to
727 * bind it. Free dynamic mappings have their interrupt set as
728 * SDEI_DYN_IRQ.
729 */
730 map = find_event_map_by_intr(SDEI_DYN_IRQ, shared_mapping);
731 if (map == NULL)
732 return SDEI_ENOMEM;
733
734 /* The returned mapping must be dynamic */
735 assert(is_map_dynamic(map));
736
737 /*
738 * We cannot assert for bound maps here, as we might be racing
739 * with another bind.
740 */
741
742 /* The requested interrupt must already belong to NS */
743 if (plat_ic_get_interrupt_type(intr_num) != INTR_TYPE_NS)
744 return SDEI_EDENY;
745
746 /*
747 * Interrupt programming and ownership transfer are deferred
748 * until register.
749 */
750
751 sdei_map_lock(map);
752 if (!is_map_bound(map)) {
753 map->intr = intr_num;
754 set_map_bound(map);
755 retry = false;
756 }
757 sdei_map_unlock(map);
758 } while (retry);
759
760 return map->ev_num;
761 }
762
763 /* Release a bound SDEI event previously to an interrupt */
sdei_interrupt_release(int ev_num)764 static int sdei_interrupt_release(int ev_num)
765 {
766 int ret = 0;
767 sdei_ev_map_t *map;
768 sdei_entry_t *se;
769
770 /* Check if valid event number */
771 map = find_event_map(ev_num);
772 if (map == NULL)
773 return SDEI_EINVAL;
774
775 if (!is_map_dynamic(map))
776 return SDEI_EINVAL;
777
778 se = get_event_entry(map);
779
780 sdei_map_lock(map);
781
782 /* Event must have been unregistered before release */
783 if (map->reg_count != 0) {
784 ret = SDEI_EDENY;
785 goto finish;
786 }
787
788 /*
789 * Interrupt release never causes the state to change. We only check
790 * whether it's permissible or not.
791 */
792 if (!can_sdei_state_trans(se, DO_RELEASE)) {
793 ret = SDEI_EDENY;
794 goto finish;
795 }
796
797 if (is_map_bound(map)) {
798 /*
799 * Deny release if the interrupt is active, which means it's
800 * probably being acknowledged and handled elsewhere.
801 */
802 if (plat_ic_get_interrupt_active(map->intr) != 0U) {
803 ret = SDEI_EDENY;
804 goto finish;
805 }
806
807 /*
808 * Interrupt programming and ownership transfer are already done
809 * during unregister.
810 */
811
812 map->intr = SDEI_DYN_IRQ;
813 clr_map_bound(map);
814 } else {
815 SDEI_LOG("Error release bound:%d cnt:%d\n", is_map_bound(map),
816 map->reg_count);
817 ret = SDEI_EINVAL;
818 }
819
820 finish:
821 sdei_map_unlock(map);
822
823 return ret;
824 }
825
826 /* Perform reset of private SDEI events */
sdei_private_reset(void)827 static int sdei_private_reset(void)
828 {
829 sdei_ev_map_t *map;
830 int ret = 0, final_ret = 0;
831 unsigned int i;
832
833 /* Unregister all private events */
834 for_each_private_map(i, map) {
835 /*
836 * The unregister can fail if the event is not registered, which
837 * is allowed, and a deny will be returned. But if the event is
838 * running or unregister pending, the call fails.
839 */
840 ret = sdei_event_unregister(map->ev_num);
841 if ((ret == SDEI_EPEND) && (final_ret == 0))
842 final_ret = SDEI_EDENY;
843 }
844
845 return final_ret;
846 }
847
848 /* Perform reset of shared SDEI events */
sdei_shared_reset(void)849 static int sdei_shared_reset(void)
850 {
851 const sdei_mapping_t *mapping;
852 sdei_ev_map_t *map;
853 int ret = 0, final_ret = 0;
854 unsigned int i, j;
855
856 /* Unregister all shared events */
857 for_each_shared_map(i, map) {
858 /*
859 * The unregister can fail if the event is not registered, which
860 * is allowed, and a deny will be returned. But if the event is
861 * running or unregister pending, the call fails.
862 */
863 ret = sdei_event_unregister(map->ev_num);
864 if ((ret == SDEI_EPEND) && (final_ret == 0))
865 final_ret = SDEI_EDENY;
866 }
867
868 if (final_ret != 0)
869 return final_ret;
870
871 /*
872 * Loop through both private and shared mappings, and release all
873 * bindings.
874 */
875 for_each_mapping_type(i, mapping) {
876 iterate_mapping(mapping, j, map) {
877 /*
878 * Release bindings for mappings that are dynamic and
879 * bound.
880 */
881 if (is_map_dynamic(map) && is_map_bound(map)) {
882 /*
883 * Any failure to release would mean there is at
884 * least a PE registered for the event.
885 */
886 ret = sdei_interrupt_release(map->ev_num);
887 if ((ret != 0) && (final_ret == 0))
888 final_ret = ret;
889 }
890 }
891 }
892
893 return final_ret;
894 }
895
896 /* Send a signal to another SDEI client PE */
sdei_signal(int ev_num,uint64_t target_pe)897 static int sdei_signal(int ev_num, uint64_t target_pe)
898 {
899 sdei_ev_map_t *map;
900
901 /* Only event 0 can be signalled */
902 if (ev_num != SDEI_EVENT_0)
903 return SDEI_EINVAL;
904
905 /* Find mapping for event 0 */
906 map = find_event_map(SDEI_EVENT_0);
907 if (map == NULL)
908 return SDEI_EINVAL;
909
910 /* The event must be signalable */
911 if (!is_event_signalable(map))
912 return SDEI_EINVAL;
913
914 /* Validate target */
915 if (plat_core_pos_by_mpidr(target_pe) < 0)
916 return SDEI_EINVAL;
917
918 /* Raise SGI. Platform will validate target_pe */
919 plat_ic_raise_el3_sgi((int) map->intr, (u_register_t) target_pe);
920
921 return 0;
922 }
923
924 /* Query SDEI dispatcher features */
sdei_features(unsigned int feature)925 static uint64_t sdei_features(unsigned int feature)
926 {
927 if (feature == SDEI_FEATURE_BIND_SLOTS) {
928 return FEATURE_BIND_SLOTS(num_dyn_priv_slots,
929 num_dyn_shrd_slots);
930 }
931
932 return (uint64_t) SDEI_EINVAL;
933 }
934
935 /* SDEI top level handler for servicing SMCs */
sdei_smc_handler(uint32_t smc_fid,uint64_t x1,uint64_t x2,uint64_t x3,uint64_t x4,void * cookie,void * handle,uint64_t flags)936 uint64_t sdei_smc_handler(uint32_t smc_fid,
937 uint64_t x1,
938 uint64_t x2,
939 uint64_t x3,
940 uint64_t x4,
941 void *cookie,
942 void *handle,
943 uint64_t flags)
944 {
945
946 uint64_t x5;
947 unsigned int ss = (unsigned int) get_interrupt_src_ss(flags);
948 int64_t ret;
949 bool resume = false;
950 cpu_context_t *ctx = handle;
951 int ev_num = (int) x1;
952
953 if (ss != NON_SECURE)
954 SMC_RET1(ctx, SMC_UNK);
955
956 /* Verify the caller EL */
957 if (GET_EL(read_spsr_el3()) != sdei_client_el())
958 SMC_RET1(ctx, SMC_UNK);
959
960 switch (smc_fid) {
961 case SDEI_VERSION:
962 SDEI_LOG("> VER\n");
963 ret = (int64_t) sdei_version();
964 SDEI_LOG("< VER:%llx\n", ret);
965 SMC_RET1(ctx, ret);
966
967 case SDEI_EVENT_REGISTER:
968 x5 = SMC_GET_GP(ctx, CTX_GPREG_X5);
969 SDEI_LOG("> REG(n:%d e:%llx a:%llx f:%x m:%llx)\n", ev_num,
970 x2, x3, (int) x4, x5);
971 ret = sdei_event_register(ev_num, x2, x3, x4, x5);
972 SDEI_LOG("< REG:%lld\n", ret);
973 SMC_RET1(ctx, ret);
974
975 case SDEI_EVENT_ENABLE:
976 SDEI_LOG("> ENABLE(n:%d)\n", (int) x1);
977 ret = sdei_event_enable(ev_num);
978 SDEI_LOG("< ENABLE:%lld\n", ret);
979 SMC_RET1(ctx, ret);
980
981 case SDEI_EVENT_DISABLE:
982 SDEI_LOG("> DISABLE(n:%d)\n", ev_num);
983 ret = sdei_event_disable(ev_num);
984 SDEI_LOG("< DISABLE:%lld\n", ret);
985 SMC_RET1(ctx, ret);
986
987 case SDEI_EVENT_CONTEXT:
988 SDEI_LOG("> CTX(p:%d):%lx\n", (int) x1, read_mpidr_el1());
989 ret = sdei_event_context(ctx, (unsigned int) x1);
990 SDEI_LOG("< CTX:%lld\n", ret);
991 SMC_RET1(ctx, ret);
992
993 case SDEI_EVENT_COMPLETE_AND_RESUME:
994 resume = true;
995 /* Fallthrough */
996
997 case SDEI_EVENT_COMPLETE:
998 SDEI_LOG("> COMPLETE(r:%u sta/ep:%llx):%lx\n",
999 (unsigned int) resume, x1, read_mpidr_el1());
1000 ret = sdei_event_complete(resume, x1);
1001 SDEI_LOG("< COMPLETE:%llx\n", ret);
1002
1003 /*
1004 * Set error code only if the call failed. If the call
1005 * succeeded, we discard the dispatched context, and restore the
1006 * interrupted context to a pristine condition, and therefore
1007 * shouldn't be modified. We don't return to the caller in this
1008 * case anyway.
1009 */
1010 if (ret != 0)
1011 SMC_RET1(ctx, ret);
1012
1013 SMC_RET0(ctx);
1014
1015 case SDEI_EVENT_STATUS:
1016 SDEI_LOG("> STAT(n:%d)\n", ev_num);
1017 ret = sdei_event_status(ev_num);
1018 SDEI_LOG("< STAT:%lld\n", ret);
1019 SMC_RET1(ctx, ret);
1020
1021 case SDEI_EVENT_GET_INFO:
1022 SDEI_LOG("> INFO(n:%d, %d)\n", ev_num, (int) x2);
1023 ret = sdei_event_get_info(ev_num, (int) x2);
1024 SDEI_LOG("< INFO:%lld\n", ret);
1025 SMC_RET1(ctx, ret);
1026
1027 case SDEI_EVENT_UNREGISTER:
1028 SDEI_LOG("> UNREG(n:%d)\n", ev_num);
1029 ret = sdei_event_unregister(ev_num);
1030 SDEI_LOG("< UNREG:%lld\n", ret);
1031 SMC_RET1(ctx, ret);
1032
1033 case SDEI_PE_UNMASK:
1034 SDEI_LOG("> UNMASK:%lx\n", read_mpidr_el1());
1035 sdei_pe_unmask();
1036 SDEI_LOG("< UNMASK:%d\n", 0);
1037 SMC_RET1(ctx, 0);
1038
1039 case SDEI_PE_MASK:
1040 SDEI_LOG("> MASK:%lx\n", read_mpidr_el1());
1041 ret = sdei_pe_mask();
1042 SDEI_LOG("< MASK:%lld\n", ret);
1043 SMC_RET1(ctx, ret);
1044
1045 case SDEI_INTERRUPT_BIND:
1046 SDEI_LOG("> BIND(%d)\n", (int) x1);
1047 ret = sdei_interrupt_bind((unsigned int) x1);
1048 SDEI_LOG("< BIND:%lld\n", ret);
1049 SMC_RET1(ctx, ret);
1050
1051 case SDEI_INTERRUPT_RELEASE:
1052 SDEI_LOG("> REL(%d)\n", ev_num);
1053 ret = sdei_interrupt_release(ev_num);
1054 SDEI_LOG("< REL:%lld\n", ret);
1055 SMC_RET1(ctx, ret);
1056
1057 case SDEI_SHARED_RESET:
1058 SDEI_LOG("> S_RESET():%lx\n", read_mpidr_el1());
1059 ret = sdei_shared_reset();
1060 SDEI_LOG("< S_RESET:%lld\n", ret);
1061 SMC_RET1(ctx, ret);
1062
1063 case SDEI_PRIVATE_RESET:
1064 SDEI_LOG("> P_RESET():%lx\n", read_mpidr_el1());
1065 ret = sdei_private_reset();
1066 SDEI_LOG("< P_RESET:%lld\n", ret);
1067 SMC_RET1(ctx, ret);
1068
1069 case SDEI_EVENT_ROUTING_SET:
1070 SDEI_LOG("> ROUTE_SET(n:%d f:%llx aff:%llx)\n", ev_num, x2, x3);
1071 ret = sdei_event_routing_set(ev_num, x2, x3);
1072 SDEI_LOG("< ROUTE_SET:%lld\n", ret);
1073 SMC_RET1(ctx, ret);
1074
1075 case SDEI_FEATURES:
1076 SDEI_LOG("> FTRS(f:%llx)\n", x1);
1077 ret = (int64_t) sdei_features((unsigned int) x1);
1078 SDEI_LOG("< FTRS:%llx\n", ret);
1079 SMC_RET1(ctx, ret);
1080
1081 case SDEI_EVENT_SIGNAL:
1082 SDEI_LOG("> SIGNAL(e:%d t:%llx)\n", ev_num, x2);
1083 ret = sdei_signal(ev_num, x2);
1084 SDEI_LOG("< SIGNAL:%lld\n", ret);
1085 SMC_RET1(ctx, ret);
1086
1087 default:
1088 /* Do nothing in default case */
1089 break;
1090 }
1091
1092 WARN("Unimplemented SDEI Call: 0x%x\n", smc_fid);
1093 SMC_RET1(ctx, SMC_UNK);
1094 }
1095
1096 /* Subscribe to PSCI CPU on to initialize per-CPU SDEI configuration */
1097 SUBSCRIBE_TO_EVENT(psci_cpu_on_finish, sdei_cpu_on_init);
1098
1099 /* Subscribe to PSCI CPU suspend finisher for per-CPU configuration */
1100 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, sdei_cpu_wakeup_init);
1101