1 /*
2 * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <cdefs.h>
9 #include <inttypes.h>
10 #include <stdbool.h>
11 #include <stdint.h>
12
13 #include "../amu_private.h"
14 #include <arch.h>
15 #include <arch_features.h>
16 #include <arch_helpers.h>
17 #include <common/debug.h>
18 #include <lib/el3_runtime/pubsub_events.h>
19 #include <lib/extensions/amu.h>
20
21 #include <plat/common/platform.h>
22
23 #if ENABLE_AMU_FCONF
24 # include <lib/fconf/fconf.h>
25 # include <lib/fconf/fconf_amu_getter.h>
26 #endif
27
28 #if ENABLE_MPMM
29 # include <lib/mpmm/mpmm.h>
30 #endif
31
32 struct amu_ctx {
33 uint64_t group0_cnts[AMU_GROUP0_MAX_COUNTERS];
34 #if ENABLE_AMU_AUXILIARY_COUNTERS
35 uint64_t group1_cnts[AMU_GROUP1_MAX_COUNTERS];
36 #endif
37
38 /* Architected event counter 1 does not have an offset register */
39 uint64_t group0_voffsets[AMU_GROUP0_MAX_COUNTERS - 1U];
40 #if ENABLE_AMU_AUXILIARY_COUNTERS
41 uint64_t group1_voffsets[AMU_GROUP1_MAX_COUNTERS];
42 #endif
43
44 uint16_t group0_enable;
45 #if ENABLE_AMU_AUXILIARY_COUNTERS
46 uint16_t group1_enable;
47 #endif
48 };
49
50 static struct amu_ctx amu_ctxs_[PLATFORM_CORE_COUNT];
51
52 CASSERT((sizeof(amu_ctxs_[0].group0_enable) * CHAR_BIT) <= AMU_GROUP0_MAX_COUNTERS,
53 amu_ctx_group0_enable_cannot_represent_all_group0_counters);
54
55 #if ENABLE_AMU_AUXILIARY_COUNTERS
56 CASSERT((sizeof(amu_ctxs_[0].group1_enable) * CHAR_BIT) <= AMU_GROUP1_MAX_COUNTERS,
57 amu_ctx_group1_enable_cannot_represent_all_group1_counters);
58 #endif
59
read_id_aa64pfr0_el1_amu(void)60 static inline __unused uint64_t read_id_aa64pfr0_el1_amu(void)
61 {
62 return (read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) &
63 ID_AA64PFR0_AMU_MASK;
64 }
65
read_hcr_el2_amvoffen(void)66 static inline __unused uint64_t read_hcr_el2_amvoffen(void)
67 {
68 return (read_hcr_el2() & HCR_AMVOFFEN_BIT) >>
69 HCR_AMVOFFEN_SHIFT;
70 }
71
write_cptr_el2_tam(uint64_t value)72 static inline __unused void write_cptr_el2_tam(uint64_t value)
73 {
74 write_cptr_el2((read_cptr_el2() & ~CPTR_EL2_TAM_BIT) |
75 ((value << CPTR_EL2_TAM_SHIFT) & CPTR_EL2_TAM_BIT));
76 }
77
write_cptr_el3_tam(cpu_context_t * ctx,uint64_t tam)78 static inline __unused void write_cptr_el3_tam(cpu_context_t *ctx, uint64_t tam)
79 {
80 uint64_t value = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3);
81
82 value &= ~TAM_BIT;
83 value |= (tam << TAM_SHIFT) & TAM_BIT;
84
85 write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, value);
86 }
87
write_hcr_el2_amvoffen(uint64_t value)88 static inline __unused void write_hcr_el2_amvoffen(uint64_t value)
89 {
90 write_hcr_el2((read_hcr_el2() & ~HCR_AMVOFFEN_BIT) |
91 ((value << HCR_AMVOFFEN_SHIFT) & HCR_AMVOFFEN_BIT));
92 }
93
write_amcr_el0_cg1rz(uint64_t value)94 static inline __unused void write_amcr_el0_cg1rz(uint64_t value)
95 {
96 write_amcr_el0((read_amcr_el0() & ~AMCR_CG1RZ_BIT) |
97 ((value << AMCR_CG1RZ_SHIFT) & AMCR_CG1RZ_BIT));
98 }
99
read_amcfgr_el0_ncg(void)100 static inline __unused uint64_t read_amcfgr_el0_ncg(void)
101 {
102 return (read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT) &
103 AMCFGR_EL0_NCG_MASK;
104 }
105
read_amcgcr_el0_cg0nc(void)106 static inline __unused uint64_t read_amcgcr_el0_cg0nc(void)
107 {
108 return (read_amcgcr_el0() >> AMCGCR_EL0_CG0NC_SHIFT) &
109 AMCGCR_EL0_CG0NC_MASK;
110 }
111
read_amcg1idr_el0_voff(void)112 static inline __unused uint64_t read_amcg1idr_el0_voff(void)
113 {
114 return (read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) &
115 AMCG1IDR_VOFF_MASK;
116 }
117
read_amcgcr_el0_cg1nc(void)118 static inline __unused uint64_t read_amcgcr_el0_cg1nc(void)
119 {
120 return (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) &
121 AMCGCR_EL0_CG1NC_MASK;
122 }
123
read_amcntenset0_el0_px(void)124 static inline __unused uint64_t read_amcntenset0_el0_px(void)
125 {
126 return (read_amcntenset0_el0() >> AMCNTENSET0_EL0_Pn_SHIFT) &
127 AMCNTENSET0_EL0_Pn_MASK;
128 }
129
read_amcntenset1_el0_px(void)130 static inline __unused uint64_t read_amcntenset1_el0_px(void)
131 {
132 return (read_amcntenset1_el0() >> AMCNTENSET1_EL0_Pn_SHIFT) &
133 AMCNTENSET1_EL0_Pn_MASK;
134 }
135
write_amcntenset0_el0_px(uint64_t px)136 static inline __unused void write_amcntenset0_el0_px(uint64_t px)
137 {
138 uint64_t value = read_amcntenset0_el0();
139
140 value &= ~AMCNTENSET0_EL0_Pn_MASK;
141 value |= (px << AMCNTENSET0_EL0_Pn_SHIFT) & AMCNTENSET0_EL0_Pn_MASK;
142
143 write_amcntenset0_el0(value);
144 }
145
write_amcntenset1_el0_px(uint64_t px)146 static inline __unused void write_amcntenset1_el0_px(uint64_t px)
147 {
148 uint64_t value = read_amcntenset1_el0();
149
150 value &= ~AMCNTENSET1_EL0_Pn_MASK;
151 value |= (px << AMCNTENSET1_EL0_Pn_SHIFT) & AMCNTENSET1_EL0_Pn_MASK;
152
153 write_amcntenset1_el0(value);
154 }
155
write_amcntenclr0_el0_px(uint64_t px)156 static inline __unused void write_amcntenclr0_el0_px(uint64_t px)
157 {
158 uint64_t value = read_amcntenclr0_el0();
159
160 value &= ~AMCNTENCLR0_EL0_Pn_MASK;
161 value |= (px << AMCNTENCLR0_EL0_Pn_SHIFT) & AMCNTENCLR0_EL0_Pn_MASK;
162
163 write_amcntenclr0_el0(value);
164 }
165
write_amcntenclr1_el0_px(uint64_t px)166 static inline __unused void write_amcntenclr1_el0_px(uint64_t px)
167 {
168 uint64_t value = read_amcntenclr1_el0();
169
170 value &= ~AMCNTENCLR1_EL0_Pn_MASK;
171 value |= (px << AMCNTENCLR1_EL0_Pn_SHIFT) & AMCNTENCLR1_EL0_Pn_MASK;
172
173 write_amcntenclr1_el0(value);
174 }
175
amu_supported(void)176 static __unused bool amu_supported(void)
177 {
178 return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1;
179 }
180
amu_v1p1_supported(void)181 static __unused bool amu_v1p1_supported(void)
182 {
183 return read_id_aa64pfr0_el1_amu() >= ID_AA64PFR0_AMU_V1P1;
184 }
185
186 #if ENABLE_AMU_AUXILIARY_COUNTERS
amu_group1_supported(void)187 static __unused bool amu_group1_supported(void)
188 {
189 return read_amcfgr_el0_ncg() > 0U;
190 }
191 #endif
192
193 /*
194 * Enable counters. This function is meant to be invoked by the context
195 * management library before exiting from EL3.
196 */
amu_enable(bool el2_unused,cpu_context_t * ctx)197 void amu_enable(bool el2_unused, cpu_context_t *ctx)
198 {
199 uint64_t id_aa64pfr0_el1_amu; /* AMU version */
200
201 uint64_t amcfgr_el0_ncg; /* Number of counter groups */
202 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */
203
204 uint64_t amcntenset0_el0_px = 0x0; /* Group 0 enable mask */
205 uint64_t amcntenset1_el0_px = 0x0; /* Group 1 enable mask */
206
207 id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu();
208 if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) {
209 /*
210 * If the AMU is unsupported, nothing needs to be done.
211 */
212
213 return;
214 }
215
216 if (el2_unused) {
217 /*
218 * CPTR_EL2.TAM: Set to zero so any accesses to the Activity
219 * Monitor registers do not trap to EL2.
220 */
221 write_cptr_el2_tam(0U);
222 }
223
224 /*
225 * Retrieve and update the CPTR_EL3 value from the context mentioned
226 * in 'ctx'. Set CPTR_EL3.TAM to zero so that any accesses to
227 * the Activity Monitor registers do not trap to EL3.
228 */
229 write_cptr_el3_tam(ctx, 0U);
230
231 /*
232 * Retrieve the number of architected counters. All of these counters
233 * are enabled by default.
234 */
235
236 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc();
237 amcntenset0_el0_px = (UINT64_C(1) << (amcgcr_el0_cg0nc)) - 1U;
238
239 assert(amcgcr_el0_cg0nc <= AMU_AMCGCR_CG0NC_MAX);
240
241 /*
242 * The platform may opt to enable specific auxiliary counters. This can
243 * be done via the common FCONF getter, or via the platform-implemented
244 * function.
245 */
246
247 #if ENABLE_AMU_AUXILIARY_COUNTERS
248 const struct amu_topology *topology;
249
250 #if ENABLE_AMU_FCONF
251 topology = FCONF_GET_PROPERTY(amu, config, topology);
252 #else
253 topology = plat_amu_topology();
254 #endif /* ENABLE_AMU_FCONF */
255
256 if (topology != NULL) {
257 unsigned int core_pos = plat_my_core_pos();
258
259 amcntenset1_el0_px = topology->cores[core_pos].enable;
260 } else {
261 ERROR("AMU: failed to generate AMU topology\n");
262 }
263 #endif /* ENABLE_AMU_AUXILIARY_COUNTERS */
264
265 /*
266 * Enable the requested counters.
267 */
268
269 write_amcntenset0_el0_px(amcntenset0_el0_px);
270
271 amcfgr_el0_ncg = read_amcfgr_el0_ncg();
272 if (amcfgr_el0_ncg > 0U) {
273 write_amcntenset1_el0_px(amcntenset1_el0_px);
274
275 #if !ENABLE_AMU_AUXILIARY_COUNTERS
276 VERBOSE("AMU: auxiliary counters detected but support is disabled\n");
277 #endif
278 }
279
280 /* Initialize FEAT_AMUv1p1 features if present. */
281 if (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) {
282 if (el2_unused) {
283 /*
284 * Make sure virtual offsets are disabled if EL2 not
285 * used.
286 */
287 write_hcr_el2_amvoffen(0U);
288 }
289
290 #if AMU_RESTRICT_COUNTERS
291 /*
292 * FEAT_AMUv1p1 adds a register field to restrict access to
293 * group 1 counters at all but the highest implemented EL. This
294 * is controlled with the `AMU_RESTRICT_COUNTERS` compile time
295 * flag, when set, system register reads at lower ELs return
296 * zero. Reads from the memory mapped view are unaffected.
297 */
298 VERBOSE("AMU group 1 counter access restricted.\n");
299 write_amcr_el0_cg1rz(1U);
300 #else
301 write_amcr_el0_cg1rz(0U);
302 #endif
303 }
304
305 #if ENABLE_MPMM
306 mpmm_enable();
307 #endif
308 }
309
310 /* Read the group 0 counter identified by the given `idx`. */
amu_group0_cnt_read(unsigned int idx)311 static uint64_t amu_group0_cnt_read(unsigned int idx)
312 {
313 assert(amu_supported());
314 assert(idx < read_amcgcr_el0_cg0nc());
315
316 return amu_group0_cnt_read_internal(idx);
317 }
318
319 /* Write the group 0 counter identified by the given `idx` with `val` */
amu_group0_cnt_write(unsigned int idx,uint64_t val)320 static void amu_group0_cnt_write(unsigned int idx, uint64_t val)
321 {
322 assert(amu_supported());
323 assert(idx < read_amcgcr_el0_cg0nc());
324
325 amu_group0_cnt_write_internal(idx, val);
326 isb();
327 }
328
329 /*
330 * Unlike with auxiliary counters, we cannot detect at runtime whether an
331 * architected counter supports a virtual offset. These are instead fixed
332 * according to FEAT_AMUv1p1, but this switch will need to be updated if later
333 * revisions of FEAT_AMU add additional architected counters.
334 */
amu_group0_voffset_supported(uint64_t idx)335 static bool amu_group0_voffset_supported(uint64_t idx)
336 {
337 switch (idx) {
338 case 0U:
339 case 2U:
340 case 3U:
341 return true;
342
343 case 1U:
344 return false;
345
346 default:
347 ERROR("AMU: can't set up virtual offset for unknown "
348 "architected counter %" PRIu64 "!\n", idx);
349
350 panic();
351 }
352 }
353
354 /*
355 * Read the group 0 offset register for a given index. Index must be 0, 2,
356 * or 3, the register for 1 does not exist.
357 *
358 * Using this function requires FEAT_AMUv1p1 support.
359 */
amu_group0_voffset_read(unsigned int idx)360 static uint64_t amu_group0_voffset_read(unsigned int idx)
361 {
362 assert(amu_v1p1_supported());
363 assert(idx < read_amcgcr_el0_cg0nc());
364 assert(idx != 1U);
365
366 return amu_group0_voffset_read_internal(idx);
367 }
368
369 /*
370 * Write the group 0 offset register for a given index. Index must be 0, 2, or
371 * 3, the register for 1 does not exist.
372 *
373 * Using this function requires FEAT_AMUv1p1 support.
374 */
amu_group0_voffset_write(unsigned int idx,uint64_t val)375 static void amu_group0_voffset_write(unsigned int idx, uint64_t val)
376 {
377 assert(amu_v1p1_supported());
378 assert(idx < read_amcgcr_el0_cg0nc());
379 assert(idx != 1U);
380
381 amu_group0_voffset_write_internal(idx, val);
382 isb();
383 }
384
385 #if ENABLE_AMU_AUXILIARY_COUNTERS
386 /* Read the group 1 counter identified by the given `idx` */
amu_group1_cnt_read(unsigned int idx)387 static uint64_t amu_group1_cnt_read(unsigned int idx)
388 {
389 assert(amu_supported());
390 assert(amu_group1_supported());
391 assert(idx < read_amcgcr_el0_cg1nc());
392
393 return amu_group1_cnt_read_internal(idx);
394 }
395
396 /* Write the group 1 counter identified by the given `idx` with `val` */
amu_group1_cnt_write(unsigned int idx,uint64_t val)397 static void amu_group1_cnt_write(unsigned int idx, uint64_t val)
398 {
399 assert(amu_supported());
400 assert(amu_group1_supported());
401 assert(idx < read_amcgcr_el0_cg1nc());
402
403 amu_group1_cnt_write_internal(idx, val);
404 isb();
405 }
406
407 /*
408 * Read the group 1 offset register for a given index.
409 *
410 * Using this function requires FEAT_AMUv1p1 support.
411 */
amu_group1_voffset_read(unsigned int idx)412 static uint64_t amu_group1_voffset_read(unsigned int idx)
413 {
414 assert(amu_v1p1_supported());
415 assert(amu_group1_supported());
416 assert(idx < read_amcgcr_el0_cg1nc());
417 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U);
418
419 return amu_group1_voffset_read_internal(idx);
420 }
421
422 /*
423 * Write the group 1 offset register for a given index.
424 *
425 * Using this function requires FEAT_AMUv1p1 support.
426 */
amu_group1_voffset_write(unsigned int idx,uint64_t val)427 static void amu_group1_voffset_write(unsigned int idx, uint64_t val)
428 {
429 assert(amu_v1p1_supported());
430 assert(amu_group1_supported());
431 assert(idx < read_amcgcr_el0_cg1nc());
432 assert((read_amcg1idr_el0_voff() & (UINT64_C(1) << idx)) != 0U);
433
434 amu_group1_voffset_write_internal(idx, val);
435 isb();
436 }
437 #endif
438
amu_context_save(const void * arg)439 static void *amu_context_save(const void *arg)
440 {
441 uint64_t i, j;
442
443 unsigned int core_pos;
444 struct amu_ctx *ctx;
445
446 uint64_t id_aa64pfr0_el1_amu; /* AMU version */
447 uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */
448 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */
449
450 #if ENABLE_AMU_AUXILIARY_COUNTERS
451 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */
452 uint64_t amcfgr_el0_ncg; /* Number of counter groups */
453 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */
454 #endif
455
456 id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu();
457 if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) {
458 return (void *)0;
459 }
460
461 core_pos = plat_my_core_pos();
462 ctx = &amu_ctxs_[core_pos];
463
464 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc();
465 hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ?
466 read_hcr_el2_amvoffen() : 0U;
467
468 #if ENABLE_AMU_AUXILIARY_COUNTERS
469 amcfgr_el0_ncg = read_amcfgr_el0_ncg();
470 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U;
471 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U;
472 #endif
473
474 /*
475 * Disable all AMU counters.
476 */
477
478 ctx->group0_enable = read_amcntenset0_el0_px();
479 write_amcntenclr0_el0_px(ctx->group0_enable);
480
481 #if ENABLE_AMU_AUXILIARY_COUNTERS
482 if (amcfgr_el0_ncg > 0U) {
483 ctx->group1_enable = read_amcntenset1_el0_px();
484 write_amcntenclr1_el0_px(ctx->group1_enable);
485 }
486 #endif
487
488 /*
489 * Save the counters to the local context.
490 */
491
492 isb(); /* Ensure counters have been stopped */
493
494 for (i = 0U; i < amcgcr_el0_cg0nc; i++) {
495 ctx->group0_cnts[i] = amu_group0_cnt_read(i);
496 }
497
498 #if ENABLE_AMU_AUXILIARY_COUNTERS
499 for (i = 0U; i < amcgcr_el0_cg1nc; i++) {
500 ctx->group1_cnts[i] = amu_group1_cnt_read(i);
501 }
502 #endif
503
504 /*
505 * Save virtual offsets for counters that offer them.
506 */
507
508 if (hcr_el2_amvoffen != 0U) {
509 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) {
510 if (!amu_group0_voffset_supported(i)) {
511 continue; /* No virtual offset */
512 }
513
514 ctx->group0_voffsets[j++] = amu_group0_voffset_read(i);
515 }
516
517 #if ENABLE_AMU_AUXILIARY_COUNTERS
518 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) {
519 if ((amcg1idr_el0_voff >> i) & 1U) {
520 continue; /* No virtual offset */
521 }
522
523 ctx->group1_voffsets[j++] = amu_group1_voffset_read(i);
524 }
525 #endif
526 }
527
528 return (void *)0;
529 }
530
amu_context_restore(const void * arg)531 static void *amu_context_restore(const void *arg)
532 {
533 uint64_t i, j;
534
535 unsigned int core_pos;
536 struct amu_ctx *ctx;
537
538 uint64_t id_aa64pfr0_el1_amu; /* AMU version */
539
540 uint64_t hcr_el2_amvoffen; /* AMU virtual offsets enabled */
541
542 uint64_t amcfgr_el0_ncg; /* Number of counter groups */
543 uint64_t amcgcr_el0_cg0nc; /* Number of group 0 counters */
544
545 #if ENABLE_AMU_AUXILIARY_COUNTERS
546 uint64_t amcgcr_el0_cg1nc; /* Number of group 1 counters */
547 uint64_t amcg1idr_el0_voff; /* Auxiliary counters with virtual offsets */
548 #endif
549
550 id_aa64pfr0_el1_amu = read_id_aa64pfr0_el1_amu();
551 if (id_aa64pfr0_el1_amu == ID_AA64PFR0_AMU_NOT_SUPPORTED) {
552 return (void *)0;
553 }
554
555 core_pos = plat_my_core_pos();
556 ctx = &amu_ctxs_[core_pos];
557
558 amcfgr_el0_ncg = read_amcfgr_el0_ncg();
559 amcgcr_el0_cg0nc = read_amcgcr_el0_cg0nc();
560
561 hcr_el2_amvoffen = (id_aa64pfr0_el1_amu >= ID_AA64PFR0_AMU_V1P1) ?
562 read_hcr_el2_amvoffen() : 0U;
563
564 #if ENABLE_AMU_AUXILIARY_COUNTERS
565 amcgcr_el0_cg1nc = (amcfgr_el0_ncg > 0U) ? read_amcgcr_el0_cg1nc() : 0U;
566 amcg1idr_el0_voff = (hcr_el2_amvoffen != 0U) ? read_amcg1idr_el0_voff() : 0U;
567 #endif
568
569 /*
570 * Sanity check that all counters were disabled when the context was
571 * previously saved.
572 */
573
574 assert(read_amcntenset0_el0_px() == 0U);
575
576 if (amcfgr_el0_ncg > 0U) {
577 assert(read_amcntenset1_el0_px() == 0U);
578 }
579
580 /*
581 * Restore the counter values from the local context.
582 */
583
584 for (i = 0U; i < amcgcr_el0_cg0nc; i++) {
585 amu_group0_cnt_write(i, ctx->group0_cnts[i]);
586 }
587
588 #if ENABLE_AMU_AUXILIARY_COUNTERS
589 for (i = 0U; i < amcgcr_el0_cg1nc; i++) {
590 amu_group1_cnt_write(i, ctx->group1_cnts[i]);
591 }
592 #endif
593
594 /*
595 * Restore virtual offsets for counters that offer them.
596 */
597
598 if (hcr_el2_amvoffen != 0U) {
599 for (i = 0U, j = 0U; i < amcgcr_el0_cg0nc; i++) {
600 if (!amu_group0_voffset_supported(i)) {
601 continue; /* No virtual offset */
602 }
603
604 amu_group0_voffset_write(i, ctx->group0_voffsets[j++]);
605 }
606
607 #if ENABLE_AMU_AUXILIARY_COUNTERS
608 for (i = 0U, j = 0U; i < amcgcr_el0_cg1nc; i++) {
609 if ((amcg1idr_el0_voff >> i) & 1U) {
610 continue; /* No virtual offset */
611 }
612
613 amu_group1_voffset_write(i, ctx->group1_voffsets[j++]);
614 }
615 #endif
616 }
617
618 /*
619 * Re-enable counters that were disabled during context save.
620 */
621
622 write_amcntenset0_el0_px(ctx->group0_enable);
623
624 #if ENABLE_AMU_AUXILIARY_COUNTERS
625 if (amcfgr_el0_ncg > 0) {
626 write_amcntenset1_el0_px(ctx->group1_enable);
627 }
628 #endif
629
630 #if ENABLE_MPMM
631 mpmm_enable();
632 #endif
633
634 return (void *)0;
635 }
636
637 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save);
638 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore);
639