1 /*
2 * Copyright (c) 2017-2020, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <arch_helpers.h>
8 #include <assert.h>
9 #include <inttypes.h>
10 #include <stddef.h>
11 #include <stdint.h>
12 #include <string.h>
13
14 #include <bl31/bl31.h>
15 #include <bl31/ehf.h>
16 #include <bl31/interrupt_mgmt.h>
17 #include <common/bl_common.h>
18 #include <common/debug.h>
19 #include <common/runtime_svc.h>
20 #include <context.h>
21 #include <lib/cassert.h>
22 #include <lib/el3_runtime/pubsub.h>
23 #include <lib/utils.h>
24 #include <plat/common/platform.h>
25 #include <services/sdei.h>
26
27 #include "sdei_private.h"
28
29 #define MAJOR_VERSION 1ULL
30 #define MINOR_VERSION 0ULL
31 #define VENDOR_VERSION 0ULL
32
33 #define MAKE_SDEI_VERSION(_major, _minor, _vendor) \
34 ((((_major)) << 48ULL) | (((_minor)) << 32ULL) | (_vendor))
35
36 #define LOWEST_INTR_PRIORITY 0xff
37
38 #define is_valid_affinity(_mpidr) (plat_core_pos_by_mpidr(_mpidr) >= 0)
39
40 CASSERT(PLAT_SDEI_CRITICAL_PRI < PLAT_SDEI_NORMAL_PRI,
41 sdei_critical_must_have_higher_priority);
42
43 static unsigned int num_dyn_priv_slots, num_dyn_shrd_slots;
44
45 /* Initialise SDEI map entries */
init_map(sdei_ev_map_t * map)46 static void init_map(sdei_ev_map_t *map)
47 {
48 map->reg_count = 0;
49 }
50
51 /* Convert mapping to SDEI class */
map_to_class(sdei_ev_map_t * map)52 static sdei_class_t map_to_class(sdei_ev_map_t *map)
53 {
54 return is_event_critical(map) ? SDEI_CRITICAL : SDEI_NORMAL;
55 }
56
57 /* Clear SDEI event entries except state */
clear_event_entries(sdei_entry_t * se)58 static void clear_event_entries(sdei_entry_t *se)
59 {
60 se->ep = 0;
61 se->arg = 0;
62 se->affinity = 0;
63 se->reg_flags = 0;
64 }
65
66 /* Perform CPU-specific state initialisation */
sdei_cpu_on_init(const void * arg)67 static void *sdei_cpu_on_init(const void *arg)
68 {
69 unsigned int i;
70 sdei_ev_map_t *map;
71 sdei_entry_t *se;
72
73 /* Initialize private mappings on this CPU */
74 for_each_private_map(i, map) {
75 se = get_event_entry(map);
76 clear_event_entries(se);
77 se->state = 0;
78 }
79
80 SDEI_LOG("Private events initialized on %lx\n", read_mpidr_el1());
81
82 /* All PEs start with SDEI events masked */
83 (void) sdei_pe_mask();
84
85 return NULL;
86 }
87
88 /* CPU initialisation after wakeup from suspend */
sdei_cpu_wakeup_init(const void * arg)89 static void *sdei_cpu_wakeup_init(const void *arg)
90 {
91 SDEI_LOG("Events masked on %lx\n", read_mpidr_el1());
92
93 /* All PEs wake up with SDEI events masked */
94 sdei_pe_mask();
95
96 return 0;
97 }
98
99 /* Initialise an SDEI class */
sdei_class_init(sdei_class_t class)100 static void sdei_class_init(sdei_class_t class)
101 {
102 unsigned int i;
103 bool zero_found __unused = false;
104 int ev_num_so_far __unused;
105 sdei_ev_map_t *map;
106
107 /* Sanity check and configuration of shared events */
108 ev_num_so_far = -1;
109 for_each_shared_map(i, map) {
110 #if ENABLE_ASSERTIONS
111 /* Ensure mappings are sorted */
112 assert((ev_num_so_far < 0) || (map->ev_num > ev_num_so_far));
113
114 ev_num_so_far = map->ev_num;
115
116 /* Event 0 must not be shared */
117 assert(map->ev_num != SDEI_EVENT_0);
118
119 /* Check for valid event */
120 assert(map->ev_num >= 0);
121
122 /* Make sure it's a shared event */
123 assert(is_event_shared(map));
124
125 /* No shared mapping should have signalable property */
126 assert(!is_event_signalable(map));
127
128 /* Shared mappings can't be explicit */
129 assert(!is_map_explicit(map));
130 #endif
131
132 /* Skip initializing the wrong priority */
133 if (map_to_class(map) != class)
134 continue;
135
136 /* Platform events are always bound, so set the bound flag */
137 if (is_map_dynamic(map)) {
138 assert(map->intr == SDEI_DYN_IRQ);
139 assert(is_event_normal(map));
140 num_dyn_shrd_slots++;
141 } else {
142 /* Shared mappings must be bound to shared interrupt */
143 assert(plat_ic_is_spi(map->intr) != 0);
144 set_map_bound(map);
145 }
146
147 init_map(map);
148 }
149
150 /* Sanity check and configuration of private events for this CPU */
151 ev_num_so_far = -1;
152 for_each_private_map(i, map) {
153 #if ENABLE_ASSERTIONS
154 /* Ensure mappings are sorted */
155 assert((ev_num_so_far < 0) || (map->ev_num > ev_num_so_far));
156
157 ev_num_so_far = map->ev_num;
158
159 if (map->ev_num == SDEI_EVENT_0) {
160 zero_found = true;
161
162 /* Event 0 must be a Secure SGI */
163 assert(is_secure_sgi(map->intr));
164
165 /*
166 * Event 0 can have only have signalable flag (apart
167 * from being private
168 */
169 assert(map->map_flags == (SDEI_MAPF_SIGNALABLE |
170 SDEI_MAPF_PRIVATE));
171 } else {
172 /* No other mapping should have signalable property */
173 assert(!is_event_signalable(map));
174 }
175
176 /* Check for valid event */
177 assert(map->ev_num >= 0);
178
179 /* Make sure it's a private event */
180 assert(is_event_private(map));
181
182 /*
183 * Other than priority, explicit events can only have explicit
184 * and private flags set.
185 */
186 if (is_map_explicit(map)) {
187 assert((map->map_flags | SDEI_MAPF_CRITICAL) ==
188 (SDEI_MAPF_EXPLICIT | SDEI_MAPF_PRIVATE
189 | SDEI_MAPF_CRITICAL));
190 }
191 #endif
192
193 /* Skip initializing the wrong priority */
194 if (map_to_class(map) != class)
195 continue;
196
197 /* Platform events are always bound, so set the bound flag */
198 if (map->ev_num != SDEI_EVENT_0) {
199 if (is_map_dynamic(map)) {
200 assert(map->intr == SDEI_DYN_IRQ);
201 assert(is_event_normal(map));
202 num_dyn_priv_slots++;
203 } else if (is_map_explicit(map)) {
204 /*
205 * Explicit mappings don't have a backing
206 * SDEI interrupt, but verify that anyway.
207 */
208 assert(map->intr == SDEI_DYN_IRQ);
209 } else {
210 /*
211 * Private mappings must be bound to private
212 * interrupt.
213 */
214 assert(plat_ic_is_ppi((unsigned) map->intr) != 0);
215 set_map_bound(map);
216 }
217 }
218
219 init_map(map);
220 }
221
222 /* Ensure event 0 is in the mapping */
223 assert(zero_found);
224
225 (void) sdei_cpu_on_init(NULL);
226 }
227
228 /* SDEI dispatcher initialisation */
sdei_init(void)229 void sdei_init(void)
230 {
231 plat_sdei_setup();
232 sdei_class_init(SDEI_CRITICAL);
233 sdei_class_init(SDEI_NORMAL);
234
235 /* Register priority level handlers */
236 ehf_register_priority_handler(PLAT_SDEI_CRITICAL_PRI,
237 sdei_intr_handler);
238 ehf_register_priority_handler(PLAT_SDEI_NORMAL_PRI,
239 sdei_intr_handler);
240 }
241
242 /* Populate SDEI event entry */
set_sdei_entry(sdei_entry_t * se,uint64_t ep,uint64_t arg,unsigned int flags,uint64_t affinity)243 static void set_sdei_entry(sdei_entry_t *se, uint64_t ep, uint64_t arg,
244 unsigned int flags, uint64_t affinity)
245 {
246 assert(se != NULL);
247
248 se->ep = ep;
249 se->arg = arg;
250 se->affinity = (affinity & MPIDR_AFFINITY_MASK);
251 se->reg_flags = flags;
252 }
253
sdei_version(void)254 static uint64_t sdei_version(void)
255 {
256 return MAKE_SDEI_VERSION(MAJOR_VERSION, MINOR_VERSION, VENDOR_VERSION);
257 }
258
259 /* Validate flags and MPIDR values for REGISTER and ROUTING_SET calls */
validate_flags(uint64_t flags,uint64_t mpidr)260 static int validate_flags(uint64_t flags, uint64_t mpidr)
261 {
262 /* Validate flags */
263 switch (flags) {
264 case SDEI_REGF_RM_PE:
265 if (!is_valid_affinity(mpidr))
266 return SDEI_EINVAL;
267 break;
268 case SDEI_REGF_RM_ANY:
269 break;
270 default:
271 /* Unknown flags */
272 return SDEI_EINVAL;
273 }
274
275 return 0;
276 }
277
278 /* Set routing of an SDEI event */
sdei_event_routing_set(int ev_num,uint64_t flags,uint64_t mpidr)279 static int sdei_event_routing_set(int ev_num, uint64_t flags, uint64_t mpidr)
280 {
281 int ret;
282 unsigned int routing;
283 sdei_ev_map_t *map;
284 sdei_entry_t *se;
285
286 ret = validate_flags(flags, mpidr);
287 if (ret != 0)
288 return ret;
289
290 /* Check if valid event number */
291 map = find_event_map(ev_num);
292 if (map == NULL)
293 return SDEI_EINVAL;
294
295 /* The event must not be private */
296 if (is_event_private(map))
297 return SDEI_EINVAL;
298
299 se = get_event_entry(map);
300
301 sdei_map_lock(map);
302
303 if (!is_map_bound(map) || is_event_private(map)) {
304 ret = SDEI_EINVAL;
305 goto finish;
306 }
307
308 if (!can_sdei_state_trans(se, DO_ROUTING)) {
309 ret = SDEI_EDENY;
310 goto finish;
311 }
312
313 /* Choose appropriate routing */
314 routing = (unsigned int) ((flags == SDEI_REGF_RM_ANY) ?
315 INTR_ROUTING_MODE_ANY : INTR_ROUTING_MODE_PE);
316
317 /* Update event registration flag */
318 se->reg_flags = (unsigned int) flags;
319 if (flags == SDEI_REGF_RM_PE) {
320 se->affinity = (mpidr & MPIDR_AFFINITY_MASK);
321 }
322
323 /*
324 * ROUTING_SET is permissible only when event composite state is
325 * 'registered, disabled, and not running'. This means that the
326 * interrupt is currently disabled, and not active.
327 */
328 plat_ic_set_spi_routing(map->intr, routing, (u_register_t) mpidr);
329
330 finish:
331 sdei_map_unlock(map);
332
333 return ret;
334 }
335
336 /* Register handler and argument for an SDEI event */
sdei_event_register(int ev_num,uint64_t ep,uint64_t arg,uint64_t flags,uint64_t mpidr)337 static int64_t sdei_event_register(int ev_num,
338 uint64_t ep,
339 uint64_t arg,
340 uint64_t flags,
341 uint64_t mpidr)
342 {
343 int ret;
344 unsigned int routing;
345 sdei_entry_t *se;
346 sdei_ev_map_t *map;
347 sdei_state_t backup_state;
348
349 if ((ep == 0U) || (plat_sdei_validate_entry_point(
350 ep, sdei_client_el()) != 0)) {
351 return SDEI_EINVAL;
352 }
353
354 ret = validate_flags(flags, mpidr);
355 if (ret != 0)
356 return ret;
357
358 /* Check if valid event number */
359 map = find_event_map(ev_num);
360 if (map == NULL)
361 return SDEI_EINVAL;
362
363 /* Private events always target the PE */
364 if (is_event_private(map)) {
365 /*
366 * SDEI internally handles private events in the same manner
367 * as public events with routing mode=RM_PE, since the routing
368 * mode flag and affinity fields are not used when registering
369 * a private event, set them here.
370 */
371 flags = SDEI_REGF_RM_PE;
372 /*
373 * Kernel may pass 0 as mpidr, as we set flags to
374 * SDEI_REGF_RM_PE, so set mpidr also.
375 */
376 mpidr = read_mpidr_el1();
377 }
378
379 se = get_event_entry(map);
380
381 /*
382 * Even though register operation is per-event (additionally for private
383 * events, registration is required individually), it has to be
384 * serialised with respect to bind/release, which are global operations.
385 * So we hold the lock throughout, unconditionally.
386 */
387 sdei_map_lock(map);
388
389 backup_state = se->state;
390 if (!can_sdei_state_trans(se, DO_REGISTER))
391 goto fallback;
392
393 /*
394 * When registering for dynamic events, make sure it's been bound
395 * already. This has to be the case as, without binding, the client
396 * can't know about the event number to register for.
397 */
398 if (is_map_dynamic(map) && !is_map_bound(map))
399 goto fallback;
400
401 if (is_event_private(map)) {
402 /* Multiple calls to register are possible for private events */
403 assert(map->reg_count >= 0);
404 } else {
405 /* Only single call to register is possible for shared events */
406 assert(map->reg_count == 0);
407 }
408
409 if (is_map_bound(map)) {
410 /* Meanwhile, did any PE ACK the interrupt? */
411 if (plat_ic_get_interrupt_active(map->intr) != 0U)
412 goto fallback;
413
414 /* The interrupt must currently owned by Non-secure */
415 if (plat_ic_get_interrupt_type(map->intr) != INTR_TYPE_NS)
416 goto fallback;
417
418 /*
419 * Disable forwarding of new interrupt triggers to CPU
420 * interface.
421 */
422 plat_ic_disable_interrupt(map->intr);
423
424 /*
425 * Any events that are triggered after register and before
426 * enable should remain pending. Clear any previous interrupt
427 * triggers which are pending (except for SGIs). This has no
428 * affect on level-triggered interrupts.
429 */
430 if (ev_num != SDEI_EVENT_0)
431 plat_ic_clear_interrupt_pending(map->intr);
432
433 /* Map interrupt to EL3 and program the correct priority */
434 plat_ic_set_interrupt_type(map->intr, INTR_TYPE_EL3);
435
436 /* Program the appropriate interrupt priority */
437 plat_ic_set_interrupt_priority(map->intr, sdei_event_priority(map));
438
439 /*
440 * Set the routing mode for shared event as requested. We
441 * already ensure that shared events get bound to SPIs.
442 */
443 if (is_event_shared(map)) {
444 routing = (unsigned int) ((flags == SDEI_REGF_RM_ANY) ?
445 INTR_ROUTING_MODE_ANY : INTR_ROUTING_MODE_PE);
446 plat_ic_set_spi_routing(map->intr, routing,
447 (u_register_t) mpidr);
448 }
449 }
450
451 /* Populate event entries */
452 set_sdei_entry(se, ep, arg, (unsigned int) flags, mpidr);
453
454 /* Increment register count */
455 map->reg_count++;
456
457 sdei_map_unlock(map);
458
459 return 0;
460
461 fallback:
462 /* Reinstate previous state */
463 se->state = backup_state;
464
465 sdei_map_unlock(map);
466
467 return SDEI_EDENY;
468 }
469
470 /* Enable SDEI event */
sdei_event_enable(int ev_num)471 static int64_t sdei_event_enable(int ev_num)
472 {
473 sdei_ev_map_t *map;
474 sdei_entry_t *se;
475 int ret;
476 bool before, after;
477
478 /* Check if valid event number */
479 map = find_event_map(ev_num);
480 if (map == NULL)
481 return SDEI_EINVAL;
482
483 se = get_event_entry(map);
484 ret = SDEI_EDENY;
485
486 if (is_event_shared(map))
487 sdei_map_lock(map);
488
489 before = GET_EV_STATE(se, ENABLED);
490 if (!can_sdei_state_trans(se, DO_ENABLE))
491 goto finish;
492 after = GET_EV_STATE(se, ENABLED);
493
494 /*
495 * Enable interrupt for bound events only if there's a change in enabled
496 * state.
497 */
498 if (is_map_bound(map) && (!before && after))
499 plat_ic_enable_interrupt(map->intr);
500
501 ret = 0;
502
503 finish:
504 if (is_event_shared(map))
505 sdei_map_unlock(map);
506
507 return ret;
508 }
509
510 /* Disable SDEI event */
sdei_event_disable(int ev_num)511 static int sdei_event_disable(int ev_num)
512 {
513 sdei_ev_map_t *map;
514 sdei_entry_t *se;
515 int ret;
516 bool before, after;
517
518 /* Check if valid event number */
519 map = find_event_map(ev_num);
520 if (map == NULL)
521 return SDEI_EINVAL;
522
523 se = get_event_entry(map);
524 ret = SDEI_EDENY;
525
526 if (is_event_shared(map))
527 sdei_map_lock(map);
528
529 before = GET_EV_STATE(se, ENABLED);
530 if (!can_sdei_state_trans(se, DO_DISABLE))
531 goto finish;
532 after = GET_EV_STATE(se, ENABLED);
533
534 /*
535 * Disable interrupt for bound events only if there's a change in
536 * enabled state.
537 */
538 if (is_map_bound(map) && (before && !after))
539 plat_ic_disable_interrupt(map->intr);
540
541 ret = 0;
542
543 finish:
544 if (is_event_shared(map))
545 sdei_map_unlock(map);
546
547 return ret;
548 }
549
550 /* Query SDEI event information */
sdei_event_get_info(int ev_num,int info)551 static int64_t sdei_event_get_info(int ev_num, int info)
552 {
553 sdei_entry_t *se;
554 sdei_ev_map_t *map;
555
556 uint64_t flags;
557 bool registered;
558 uint64_t affinity;
559
560 /* Check if valid event number */
561 map = find_event_map(ev_num);
562 if (map == NULL)
563 return SDEI_EINVAL;
564
565 se = get_event_entry(map);
566
567 if (is_event_shared(map))
568 sdei_map_lock(map);
569
570 /* Sample state under lock */
571 registered = GET_EV_STATE(se, REGISTERED);
572 flags = se->reg_flags;
573 affinity = se->affinity;
574
575 if (is_event_shared(map))
576 sdei_map_unlock(map);
577
578 switch (info) {
579 case SDEI_INFO_EV_TYPE:
580 return is_event_shared(map);
581
582 case SDEI_INFO_EV_NOT_SIGNALED:
583 return !is_event_signalable(map);
584
585 case SDEI_INFO_EV_PRIORITY:
586 return is_event_critical(map);
587
588 case SDEI_INFO_EV_ROUTING_MODE:
589 if (!is_event_shared(map))
590 return SDEI_EINVAL;
591 if (!registered)
592 return SDEI_EDENY;
593 return (flags == SDEI_REGF_RM_PE);
594
595 case SDEI_INFO_EV_ROUTING_AFF:
596 if (!is_event_shared(map))
597 return SDEI_EINVAL;
598 if (!registered)
599 return SDEI_EDENY;
600 if (flags != SDEI_REGF_RM_PE)
601 return SDEI_EINVAL;
602 return affinity;
603
604 default:
605 return SDEI_EINVAL;
606 }
607 }
608
609 /* Unregister an SDEI event */
sdei_event_unregister(int ev_num)610 static int sdei_event_unregister(int ev_num)
611 {
612 int ret = 0;
613 sdei_entry_t *se;
614 sdei_ev_map_t *map;
615
616 /* Check if valid event number */
617 map = find_event_map(ev_num);
618 if (map == NULL)
619 return SDEI_EINVAL;
620
621 se = get_event_entry(map);
622
623 /*
624 * Even though unregister operation is per-event (additionally for
625 * private events, unregistration is required individually), it has to
626 * be serialised with respect to bind/release, which are global
627 * operations. So we hold the lock throughout, unconditionally.
628 */
629 sdei_map_lock(map);
630
631 if (!can_sdei_state_trans(se, DO_UNREGISTER)) {
632 /*
633 * Even if the call is invalid, and the handler is running (for
634 * example, having unregistered from a running handler earlier),
635 * return pending error code; otherwise, return deny.
636 */
637 ret = GET_EV_STATE(se, RUNNING) ? SDEI_EPEND : SDEI_EDENY;
638
639 goto finish;
640 }
641
642 map->reg_count--;
643 if (is_event_private(map)) {
644 /* Multiple calls to register are possible for private events */
645 assert(map->reg_count >= 0);
646 } else {
647 /* Only single call to register is possible for shared events */
648 assert(map->reg_count == 0);
649 }
650
651 if (is_map_bound(map)) {
652 plat_ic_disable_interrupt(map->intr);
653
654 /*
655 * Clear pending interrupt. Skip for SGIs as they may not be
656 * cleared on interrupt controllers.
657 */
658 if (ev_num != SDEI_EVENT_0)
659 plat_ic_clear_interrupt_pending(map->intr);
660
661 assert(plat_ic_get_interrupt_type(map->intr) == INTR_TYPE_EL3);
662 plat_ic_set_interrupt_type(map->intr, INTR_TYPE_NS);
663 plat_ic_set_interrupt_priority(map->intr, LOWEST_INTR_PRIORITY);
664 }
665
666 clear_event_entries(se);
667
668 /*
669 * If the handler is running at the time of unregister, return the
670 * pending error code.
671 */
672 if (GET_EV_STATE(se, RUNNING))
673 ret = SDEI_EPEND;
674
675 finish:
676 sdei_map_unlock(map);
677
678 return ret;
679 }
680
681 /* Query status of an SDEI event */
sdei_event_status(int ev_num)682 static int sdei_event_status(int ev_num)
683 {
684 sdei_ev_map_t *map;
685 sdei_entry_t *se;
686 sdei_state_t state;
687
688 /* Check if valid event number */
689 map = find_event_map(ev_num);
690 if (map == NULL)
691 return SDEI_EINVAL;
692
693 se = get_event_entry(map);
694
695 if (is_event_shared(map))
696 sdei_map_lock(map);
697
698 /* State value directly maps to the expected return format */
699 state = se->state;
700
701 if (is_event_shared(map))
702 sdei_map_unlock(map);
703
704 return (int) state;
705 }
706
707 /* Bind an SDEI event to an interrupt */
sdei_interrupt_bind(unsigned int intr_num)708 static int sdei_interrupt_bind(unsigned int intr_num)
709 {
710 sdei_ev_map_t *map;
711 bool retry = true, shared_mapping;
712
713 /* SGIs are not allowed to be bound */
714 if (plat_ic_is_sgi(intr_num) != 0)
715 return SDEI_EINVAL;
716
717 shared_mapping = (plat_ic_is_spi(intr_num) != 0);
718 do {
719 /*
720 * Bail out if there is already an event for this interrupt,
721 * either platform-defined or dynamic.
722 */
723 map = find_event_map_by_intr(intr_num, shared_mapping);
724 if (map != NULL) {
725 if (is_map_dynamic(map)) {
726 if (is_map_bound(map)) {
727 /*
728 * Dynamic event, already bound. Return
729 * event number.
730 */
731 return map->ev_num;
732 }
733 } else {
734 /* Binding non-dynamic event */
735 return SDEI_EINVAL;
736 }
737 }
738
739 /*
740 * The interrupt is not bound yet. Try to find a free slot to
741 * bind it. Free dynamic mappings have their interrupt set as
742 * SDEI_DYN_IRQ.
743 */
744 map = find_event_map_by_intr(SDEI_DYN_IRQ, shared_mapping);
745 if (map == NULL)
746 return SDEI_ENOMEM;
747
748 /* The returned mapping must be dynamic */
749 assert(is_map_dynamic(map));
750
751 /*
752 * We cannot assert for bound maps here, as we might be racing
753 * with another bind.
754 */
755
756 /* The requested interrupt must already belong to NS */
757 if (plat_ic_get_interrupt_type(intr_num) != INTR_TYPE_NS)
758 return SDEI_EDENY;
759
760 /*
761 * Interrupt programming and ownership transfer are deferred
762 * until register.
763 */
764
765 sdei_map_lock(map);
766 if (!is_map_bound(map)) {
767 map->intr = intr_num;
768 set_map_bound(map);
769 retry = false;
770 }
771 sdei_map_unlock(map);
772 } while (retry);
773
774 return map->ev_num;
775 }
776
777 /* Release a bound SDEI event previously to an interrupt */
sdei_interrupt_release(int ev_num)778 static int sdei_interrupt_release(int ev_num)
779 {
780 int ret = 0;
781 sdei_ev_map_t *map;
782 sdei_entry_t *se;
783
784 /* Check if valid event number */
785 map = find_event_map(ev_num);
786 if (map == NULL)
787 return SDEI_EINVAL;
788
789 if (!is_map_dynamic(map))
790 return SDEI_EINVAL;
791
792 se = get_event_entry(map);
793
794 sdei_map_lock(map);
795
796 /* Event must have been unregistered before release */
797 if (map->reg_count != 0) {
798 ret = SDEI_EDENY;
799 goto finish;
800 }
801
802 /*
803 * Interrupt release never causes the state to change. We only check
804 * whether it's permissible or not.
805 */
806 if (!can_sdei_state_trans(se, DO_RELEASE)) {
807 ret = SDEI_EDENY;
808 goto finish;
809 }
810
811 if (is_map_bound(map)) {
812 /*
813 * Deny release if the interrupt is active, which means it's
814 * probably being acknowledged and handled elsewhere.
815 */
816 if (plat_ic_get_interrupt_active(map->intr) != 0U) {
817 ret = SDEI_EDENY;
818 goto finish;
819 }
820
821 /*
822 * Interrupt programming and ownership transfer are already done
823 * during unregister.
824 */
825
826 map->intr = SDEI_DYN_IRQ;
827 clr_map_bound(map);
828 } else {
829 SDEI_LOG("Error release bound:%d cnt:%d\n", is_map_bound(map),
830 map->reg_count);
831 ret = SDEI_EINVAL;
832 }
833
834 finish:
835 sdei_map_unlock(map);
836
837 return ret;
838 }
839
840 /* Perform reset of private SDEI events */
sdei_private_reset(void)841 static int sdei_private_reset(void)
842 {
843 sdei_ev_map_t *map;
844 int ret = 0, final_ret = 0;
845 unsigned int i;
846
847 /* Unregister all private events */
848 for_each_private_map(i, map) {
849 /*
850 * The unregister can fail if the event is not registered, which
851 * is allowed, and a deny will be returned. But if the event is
852 * running or unregister pending, the call fails.
853 */
854 ret = sdei_event_unregister(map->ev_num);
855 if ((ret == SDEI_EPEND) && (final_ret == 0))
856 final_ret = SDEI_EDENY;
857 }
858
859 return final_ret;
860 }
861
862 /* Perform reset of shared SDEI events */
sdei_shared_reset(void)863 static int sdei_shared_reset(void)
864 {
865 const sdei_mapping_t *mapping;
866 sdei_ev_map_t *map;
867 int ret = 0, final_ret = 0;
868 unsigned int i, j;
869
870 /* Unregister all shared events */
871 for_each_shared_map(i, map) {
872 /*
873 * The unregister can fail if the event is not registered, which
874 * is allowed, and a deny will be returned. But if the event is
875 * running or unregister pending, the call fails.
876 */
877 ret = sdei_event_unregister(map->ev_num);
878 if ((ret == SDEI_EPEND) && (final_ret == 0))
879 final_ret = SDEI_EDENY;
880 }
881
882 if (final_ret != 0)
883 return final_ret;
884
885 /*
886 * Loop through both private and shared mappings, and release all
887 * bindings.
888 */
889 for_each_mapping_type(i, mapping) {
890 iterate_mapping(mapping, j, map) {
891 /*
892 * Release bindings for mappings that are dynamic and
893 * bound.
894 */
895 if (is_map_dynamic(map) && is_map_bound(map)) {
896 /*
897 * Any failure to release would mean there is at
898 * least a PE registered for the event.
899 */
900 ret = sdei_interrupt_release(map->ev_num);
901 if ((ret != 0) && (final_ret == 0))
902 final_ret = ret;
903 }
904 }
905 }
906
907 return final_ret;
908 }
909
910 /* Send a signal to another SDEI client PE */
sdei_signal(int ev_num,uint64_t target_pe)911 static int sdei_signal(int ev_num, uint64_t target_pe)
912 {
913 sdei_ev_map_t *map;
914
915 /* Only event 0 can be signalled */
916 if (ev_num != SDEI_EVENT_0)
917 return SDEI_EINVAL;
918
919 /* Find mapping for event 0 */
920 map = find_event_map(SDEI_EVENT_0);
921 if (map == NULL)
922 return SDEI_EINVAL;
923
924 /* The event must be signalable */
925 if (!is_event_signalable(map))
926 return SDEI_EINVAL;
927
928 /* Validate target */
929 if (plat_core_pos_by_mpidr(target_pe) < 0)
930 return SDEI_EINVAL;
931
932 /* Raise SGI. Platform will validate target_pe */
933 plat_ic_raise_el3_sgi((int) map->intr, (u_register_t) target_pe);
934
935 return 0;
936 }
937
938 /* Query SDEI dispatcher features */
sdei_features(unsigned int feature)939 static uint64_t sdei_features(unsigned int feature)
940 {
941 if (feature == SDEI_FEATURE_BIND_SLOTS) {
942 return FEATURE_BIND_SLOTS(num_dyn_priv_slots,
943 num_dyn_shrd_slots);
944 }
945
946 return (uint64_t) SDEI_EINVAL;
947 }
948
949 /* SDEI top level handler for servicing SMCs */
sdei_smc_handler(uint32_t smc_fid,uint64_t x1,uint64_t x2,uint64_t x3,uint64_t x4,void * cookie,void * handle,uint64_t flags)950 uint64_t sdei_smc_handler(uint32_t smc_fid,
951 uint64_t x1,
952 uint64_t x2,
953 uint64_t x3,
954 uint64_t x4,
955 void *cookie,
956 void *handle,
957 uint64_t flags)
958 {
959
960 uint64_t x5;
961 unsigned int ss = (unsigned int) get_interrupt_src_ss(flags);
962 int64_t ret;
963 bool resume = false;
964 cpu_context_t *ctx = handle;
965 int ev_num = (int) x1;
966
967 if (ss != NON_SECURE)
968 SMC_RET1(ctx, SMC_UNK);
969
970 /* Verify the caller EL */
971 if (GET_EL(read_spsr_el3()) != sdei_client_el())
972 SMC_RET1(ctx, SMC_UNK);
973
974 switch (smc_fid) {
975 case SDEI_VERSION:
976 SDEI_LOG("> VER\n");
977 ret = (int64_t) sdei_version();
978 SDEI_LOG("< VER:%" PRIx64 "\n", ret);
979 SMC_RET1(ctx, ret);
980
981 case SDEI_EVENT_REGISTER:
982 x5 = SMC_GET_GP(ctx, CTX_GPREG_X5);
983 SDEI_LOG("> REG(n:%d e:%" PRIx64 " a:%" PRIx64 " f:%x m:%" PRIx64 "\n", ev_num,
984 x2, x3, (int) x4, x5);
985 ret = sdei_event_register(ev_num, x2, x3, x4, x5);
986 SDEI_LOG("< REG:%" PRId64 "\n", ret);
987 SMC_RET1(ctx, ret);
988
989 case SDEI_EVENT_ENABLE:
990 SDEI_LOG("> ENABLE(n:%d)\n", (int) x1);
991 ret = sdei_event_enable(ev_num);
992 SDEI_LOG("< ENABLE:%" PRId64 "\n", ret);
993 SMC_RET1(ctx, ret);
994
995 case SDEI_EVENT_DISABLE:
996 SDEI_LOG("> DISABLE(n:0x%x)\n", ev_num);
997 ret = sdei_event_disable(ev_num);
998 SDEI_LOG("< DISABLE:%" PRId64 "\n", ret);
999 SMC_RET1(ctx, ret);
1000
1001 case SDEI_EVENT_CONTEXT:
1002 SDEI_LOG("> CTX(p:%d):%lx\n", (int) x1, read_mpidr_el1());
1003 ret = sdei_event_context(ctx, (unsigned int) x1);
1004 SDEI_LOG("< CTX:%" PRId64 "\n", ret);
1005 SMC_RET1(ctx, ret);
1006
1007 case SDEI_EVENT_COMPLETE_AND_RESUME:
1008 resume = true;
1009 /* Fallthrough */
1010
1011 case SDEI_EVENT_COMPLETE:
1012 SDEI_LOG("> COMPLETE(r:%u sta/ep:%" PRIx64 "):%lx\n",
1013 (unsigned int) resume, x1, read_mpidr_el1());
1014 ret = sdei_event_complete(resume, x1);
1015 SDEI_LOG("< COMPLETE:%" PRIx64 "\n", ret);
1016
1017 /*
1018 * Set error code only if the call failed. If the call
1019 * succeeded, we discard the dispatched context, and restore the
1020 * interrupted context to a pristine condition, and therefore
1021 * shouldn't be modified. We don't return to the caller in this
1022 * case anyway.
1023 */
1024 if (ret != 0)
1025 SMC_RET1(ctx, ret);
1026
1027 SMC_RET0(ctx);
1028
1029 case SDEI_EVENT_STATUS:
1030 SDEI_LOG("> STAT(n:0x%x)\n", ev_num);
1031 ret = sdei_event_status(ev_num);
1032 SDEI_LOG("< STAT:%" PRId64 "\n", ret);
1033 SMC_RET1(ctx, ret);
1034
1035 case SDEI_EVENT_GET_INFO:
1036 SDEI_LOG("> INFO(n:0x%x, %d)\n", ev_num, (int) x2);
1037 ret = sdei_event_get_info(ev_num, (int) x2);
1038 SDEI_LOG("< INFO:%" PRId64 "\n", ret);
1039 SMC_RET1(ctx, ret);
1040
1041 case SDEI_EVENT_UNREGISTER:
1042 SDEI_LOG("> UNREG(n:0x%x)\n", ev_num);
1043 ret = sdei_event_unregister(ev_num);
1044 SDEI_LOG("< UNREG:%" PRId64 "\n", ret);
1045 SMC_RET1(ctx, ret);
1046
1047 case SDEI_PE_UNMASK:
1048 SDEI_LOG("> UNMASK:%lx\n", read_mpidr_el1());
1049 sdei_pe_unmask();
1050 SDEI_LOG("< UNMASK:%d\n", 0);
1051 SMC_RET1(ctx, 0);
1052
1053 case SDEI_PE_MASK:
1054 SDEI_LOG("> MASK:%lx\n", read_mpidr_el1());
1055 ret = sdei_pe_mask();
1056 SDEI_LOG("< MASK:%" PRId64 "\n", ret);
1057 SMC_RET1(ctx, ret);
1058
1059 case SDEI_INTERRUPT_BIND:
1060 SDEI_LOG("> BIND(%d)\n", (int) x1);
1061 ret = sdei_interrupt_bind((unsigned int) x1);
1062 SDEI_LOG("< BIND:%" PRId64 "\n", ret);
1063 SMC_RET1(ctx, ret);
1064
1065 case SDEI_INTERRUPT_RELEASE:
1066 SDEI_LOG("> REL(0x%x)\n", ev_num);
1067 ret = sdei_interrupt_release(ev_num);
1068 SDEI_LOG("< REL:%" PRId64 "\n", ret);
1069 SMC_RET1(ctx, ret);
1070
1071 case SDEI_SHARED_RESET:
1072 SDEI_LOG("> S_RESET():%lx\n", read_mpidr_el1());
1073 ret = sdei_shared_reset();
1074 SDEI_LOG("< S_RESET:%" PRId64 "\n", ret);
1075 SMC_RET1(ctx, ret);
1076
1077 case SDEI_PRIVATE_RESET:
1078 SDEI_LOG("> P_RESET():%lx\n", read_mpidr_el1());
1079 ret = sdei_private_reset();
1080 SDEI_LOG("< P_RESET:%" PRId64 "\n", ret);
1081 SMC_RET1(ctx, ret);
1082
1083 case SDEI_EVENT_ROUTING_SET:
1084 SDEI_LOG("> ROUTE_SET(n:%d f:%" PRIx64 " aff:%" PRIx64 ")\n", ev_num, x2, x3);
1085 ret = sdei_event_routing_set(ev_num, x2, x3);
1086 SDEI_LOG("< ROUTE_SET:%" PRId64 "\n", ret);
1087 SMC_RET1(ctx, ret);
1088
1089 case SDEI_FEATURES:
1090 SDEI_LOG("> FTRS(f:%" PRIx64 ")\n", x1);
1091 ret = (int64_t) sdei_features((unsigned int) x1);
1092 SDEI_LOG("< FTRS:%" PRIx64 "\n", ret);
1093 SMC_RET1(ctx, ret);
1094
1095 case SDEI_EVENT_SIGNAL:
1096 SDEI_LOG("> SIGNAL(e:%d t:%" PRIx64 ")\n", ev_num, x2);
1097 ret = sdei_signal(ev_num, x2);
1098 SDEI_LOG("< SIGNAL:%" PRId64 "\n", ret);
1099 SMC_RET1(ctx, ret);
1100
1101 default:
1102 /* Do nothing in default case */
1103 break;
1104 }
1105
1106 WARN("Unimplemented SDEI Call: 0x%x\n", smc_fid);
1107 SMC_RET1(ctx, SMC_UNK);
1108 }
1109
1110 /* Subscribe to PSCI CPU on to initialize per-CPU SDEI configuration */
1111 SUBSCRIBE_TO_EVENT(psci_cpu_on_finish, sdei_cpu_on_init);
1112
1113 /* Subscribe to PSCI CPU suspend finisher for per-CPU configuration */
1114 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, sdei_cpu_wakeup_init);
1115