1 /*
2 * Copyright (c) 2017-2021, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <assert.h>
8 #include <stdbool.h>
9
10 #include <arch.h>
11 #include <arch_features.h>
12 #include <arch_helpers.h>
13
14 #include <lib/el3_runtime/pubsub_events.h>
15 #include <lib/extensions/amu.h>
16 #include <lib/extensions/amu_private.h>
17
18 #include <plat/common/platform.h>
19
20 static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT];
21
22 /*
23 * Get AMU version value from aa64pfr0.
24 * Return values
25 * ID_AA64PFR0_AMU_V1: FEAT_AMUv1 supported (introduced in ARM v8.4)
26 * ID_AA64PFR0_AMU_V1P1: FEAT_AMUv1p1 supported (introduced in ARM v8.6)
27 * ID_AA64PFR0_AMU_NOT_SUPPORTED: not supported
28 */
amu_get_version(void)29 unsigned int amu_get_version(void)
30 {
31 return (unsigned int)(read_id_aa64pfr0_el1() >> ID_AA64PFR0_AMU_SHIFT) &
32 ID_AA64PFR0_AMU_MASK;
33 }
34
35 #if AMU_GROUP1_NR_COUNTERS
36 /* Check if group 1 counters is implemented */
amu_group1_supported(void)37 bool amu_group1_supported(void)
38 {
39 uint64_t features = read_amcfgr_el0() >> AMCFGR_EL0_NCG_SHIFT;
40
41 return (features & AMCFGR_EL0_NCG_MASK) == 1U;
42 }
43 #endif
44
45 /*
46 * Enable counters. This function is meant to be invoked
47 * by the context management library before exiting from EL3.
48 */
amu_enable(bool el2_unused,cpu_context_t * ctx)49 void amu_enable(bool el2_unused, cpu_context_t *ctx)
50 {
51 uint64_t v;
52 unsigned int amu_version = amu_get_version();
53
54 if (amu_version == ID_AA64PFR0_AMU_NOT_SUPPORTED) {
55 return;
56 }
57
58 #if AMU_GROUP1_NR_COUNTERS
59 /* Check and set presence of group 1 counters */
60 if (!amu_group1_supported()) {
61 ERROR("AMU Counter Group 1 is not implemented\n");
62 panic();
63 }
64
65 /* Check number of group 1 counters */
66 uint64_t cnt_num = (read_amcgcr_el0() >> AMCGCR_EL0_CG1NC_SHIFT) &
67 AMCGCR_EL0_CG1NC_MASK;
68 VERBOSE("%s%llu. %s%u\n",
69 "Number of AMU Group 1 Counters ", cnt_num,
70 "Requested number ", AMU_GROUP1_NR_COUNTERS);
71
72 if (cnt_num < AMU_GROUP1_NR_COUNTERS) {
73 ERROR("%s%llu is less than %s%u\n",
74 "Number of AMU Group 1 Counters ", cnt_num,
75 "Requested number ", AMU_GROUP1_NR_COUNTERS);
76 panic();
77 }
78 #endif
79
80 if (el2_unused) {
81 /*
82 * CPTR_EL2.TAM: Set to zero so any accesses to
83 * the Activity Monitor registers do not trap to EL2.
84 */
85 v = read_cptr_el2();
86 v &= ~CPTR_EL2_TAM_BIT;
87 write_cptr_el2(v);
88 }
89
90 /*
91 * Retrieve and update the CPTR_EL3 value from the context mentioned
92 * in 'ctx'. Set CPTR_EL3.TAM to zero so that any accesses to
93 * the Activity Monitor registers do not trap to EL3.
94 */
95 v = read_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3);
96 v &= ~TAM_BIT;
97 write_ctx_reg(get_el3state_ctx(ctx), CTX_CPTR_EL3, v);
98
99 /* Enable group 0 counters */
100 write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK);
101
102 #if AMU_GROUP1_NR_COUNTERS
103 /* Enable group 1 counters */
104 write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK);
105 #endif
106
107 /* Initialize FEAT_AMUv1p1 features if present. */
108 if (amu_version < ID_AA64PFR0_AMU_V1P1) {
109 return;
110 }
111
112 if (el2_unused) {
113 /* Make sure virtual offsets are disabled if EL2 not used. */
114 write_hcr_el2(read_hcr_el2() & ~HCR_AMVOFFEN_BIT);
115 }
116
117 #if AMU_RESTRICT_COUNTERS
118 /*
119 * FEAT_AMUv1p1 adds a register field to restrict access to group 1
120 * counters at all but the highest implemented EL. This is controlled
121 * with the AMU_RESTRICT_COUNTERS compile time flag, when set, system
122 * register reads at lower ELs return zero. Reads from the memory
123 * mapped view are unaffected.
124 */
125 VERBOSE("AMU group 1 counter access restricted.\n");
126 write_amcr_el0(read_amcr_el0() | AMCR_CG1RZ_BIT);
127 #else
128 write_amcr_el0(read_amcr_el0() & ~AMCR_CG1RZ_BIT);
129 #endif
130 }
131
132 /* Read the group 0 counter identified by the given `idx`. */
amu_group0_cnt_read(unsigned int idx)133 uint64_t amu_group0_cnt_read(unsigned int idx)
134 {
135 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED);
136 assert(idx < AMU_GROUP0_NR_COUNTERS);
137
138 return amu_group0_cnt_read_internal(idx);
139 }
140
141 /* Write the group 0 counter identified by the given `idx` with `val` */
amu_group0_cnt_write(unsigned int idx,uint64_t val)142 void amu_group0_cnt_write(unsigned int idx, uint64_t val)
143 {
144 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED);
145 assert(idx < AMU_GROUP0_NR_COUNTERS);
146
147 amu_group0_cnt_write_internal(idx, val);
148 isb();
149 }
150
151 /*
152 * Read the group 0 offset register for a given index. Index must be 0, 2,
153 * or 3, the register for 1 does not exist.
154 *
155 * Using this function requires FEAT_AMUv1p1 support.
156 */
amu_group0_voffset_read(unsigned int idx)157 uint64_t amu_group0_voffset_read(unsigned int idx)
158 {
159 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1);
160 assert(idx < AMU_GROUP0_NR_COUNTERS);
161 assert(idx != 1U);
162
163 return amu_group0_voffset_read_internal(idx);
164 }
165
166 /*
167 * Write the group 0 offset register for a given index. Index must be 0, 2, or
168 * 3, the register for 1 does not exist.
169 *
170 * Using this function requires FEAT_AMUv1p1 support.
171 */
amu_group0_voffset_write(unsigned int idx,uint64_t val)172 void amu_group0_voffset_write(unsigned int idx, uint64_t val)
173 {
174 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1);
175 assert(idx < AMU_GROUP0_NR_COUNTERS);
176 assert(idx != 1U);
177
178 amu_group0_voffset_write_internal(idx, val);
179 isb();
180 }
181
182 #if AMU_GROUP1_NR_COUNTERS
183 /* Read the group 1 counter identified by the given `idx` */
amu_group1_cnt_read(unsigned int idx)184 uint64_t amu_group1_cnt_read(unsigned int idx)
185 {
186 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED);
187 assert(amu_group1_supported());
188 assert(idx < AMU_GROUP1_NR_COUNTERS);
189
190 return amu_group1_cnt_read_internal(idx);
191 }
192
193 /* Write the group 1 counter identified by the given `idx` with `val` */
amu_group1_cnt_write(unsigned int idx,uint64_t val)194 void amu_group1_cnt_write(unsigned int idx, uint64_t val)
195 {
196 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED);
197 assert(amu_group1_supported());
198 assert(idx < AMU_GROUP1_NR_COUNTERS);
199
200 amu_group1_cnt_write_internal(idx, val);
201 isb();
202 }
203
204 /*
205 * Read the group 1 offset register for a given index.
206 *
207 * Using this function requires FEAT_AMUv1p1 support.
208 */
amu_group1_voffset_read(unsigned int idx)209 uint64_t amu_group1_voffset_read(unsigned int idx)
210 {
211 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1);
212 assert(amu_group1_supported());
213 assert(idx < AMU_GROUP1_NR_COUNTERS);
214 assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) &
215 (1ULL << idx)) != 0ULL);
216
217 return amu_group1_voffset_read_internal(idx);
218 }
219
220 /*
221 * Write the group 1 offset register for a given index.
222 *
223 * Using this function requires FEAT_AMUv1p1 support.
224 */
amu_group1_voffset_write(unsigned int idx,uint64_t val)225 void amu_group1_voffset_write(unsigned int idx, uint64_t val)
226 {
227 assert(amu_get_version() >= ID_AA64PFR0_AMU_V1P1);
228 assert(amu_group1_supported());
229 assert(idx < AMU_GROUP1_NR_COUNTERS);
230 assert(((read_amcg1idr_el0() >> AMCG1IDR_VOFF_SHIFT) &
231 (1ULL << idx)) != 0ULL);
232
233 amu_group1_voffset_write_internal(idx, val);
234 isb();
235 }
236
237 /*
238 * Program the event type register for the given `idx` with
239 * the event number `val`
240 */
amu_group1_set_evtype(unsigned int idx,unsigned int val)241 void amu_group1_set_evtype(unsigned int idx, unsigned int val)
242 {
243 assert(amu_get_version() != ID_AA64PFR0_AMU_NOT_SUPPORTED);
244 assert(amu_group1_supported());
245 assert(idx < AMU_GROUP1_NR_COUNTERS);
246
247 amu_group1_set_evtype_internal(idx, val);
248 isb();
249 }
250 #endif /* AMU_GROUP1_NR_COUNTERS */
251
amu_context_save(const void * arg)252 static void *amu_context_save(const void *arg)
253 {
254 struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()];
255 unsigned int i;
256
257 if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) {
258 return (void *)-1;
259 }
260
261 #if AMU_GROUP1_NR_COUNTERS
262 if (!amu_group1_supported()) {
263 return (void *)-1;
264 }
265 #endif
266 /* Assert that group 0/1 counter configuration is what we expect */
267 assert(read_amcntenset0_el0() == AMU_GROUP0_COUNTERS_MASK);
268
269 #if AMU_GROUP1_NR_COUNTERS
270 assert(read_amcntenset1_el0() == AMU_GROUP1_COUNTERS_MASK);
271 #endif
272 /*
273 * Disable group 0/1 counters to avoid other observers like SCP sampling
274 * counter values from the future via the memory mapped view.
275 */
276 write_amcntenclr0_el0(AMU_GROUP0_COUNTERS_MASK);
277
278 #if AMU_GROUP1_NR_COUNTERS
279 write_amcntenclr1_el0(AMU_GROUP1_COUNTERS_MASK);
280 #endif
281 isb();
282
283 /* Save all group 0 counters */
284 for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) {
285 ctx->group0_cnts[i] = amu_group0_cnt_read(i);
286 }
287
288 /* Save group 0 virtual offsets if supported and enabled. */
289 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) &&
290 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) {
291 /* Not using a loop because count is fixed and index 1 DNE. */
292 ctx->group0_voffsets[0U] = amu_group0_voffset_read(0U);
293 ctx->group0_voffsets[1U] = amu_group0_voffset_read(2U);
294 ctx->group0_voffsets[2U] = amu_group0_voffset_read(3U);
295 }
296
297 #if AMU_GROUP1_NR_COUNTERS
298 /* Save group 1 counters */
299 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) {
300 if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) {
301 ctx->group1_cnts[i] = amu_group1_cnt_read(i);
302 }
303 }
304
305 /* Save group 1 virtual offsets if supported and enabled. */
306 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) &&
307 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) {
308 u_register_t amcg1idr = read_amcg1idr_el0() >>
309 AMCG1IDR_VOFF_SHIFT;
310 amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK;
311
312 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) {
313 if (((amcg1idr >> i) & 1ULL) != 0ULL) {
314 ctx->group1_voffsets[i] =
315 amu_group1_voffset_read(i);
316 }
317 }
318 }
319 #endif
320 return (void *)0;
321 }
322
amu_context_restore(const void * arg)323 static void *amu_context_restore(const void *arg)
324 {
325 struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()];
326 unsigned int i;
327
328 if (amu_get_version() == ID_AA64PFR0_AMU_NOT_SUPPORTED) {
329 return (void *)-1;
330 }
331
332 #if AMU_GROUP1_NR_COUNTERS
333 if (!amu_group1_supported()) {
334 return (void *)-1;
335 }
336 #endif
337 /* Counters were disabled in `amu_context_save()` */
338 assert(read_amcntenset0_el0() == 0U);
339
340 #if AMU_GROUP1_NR_COUNTERS
341 assert(read_amcntenset1_el0() == 0U);
342 #endif
343
344 /* Restore all group 0 counters */
345 for (i = 0U; i < AMU_GROUP0_NR_COUNTERS; i++) {
346 amu_group0_cnt_write(i, ctx->group0_cnts[i]);
347 }
348
349 /* Restore group 0 virtual offsets if supported and enabled. */
350 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) &&
351 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) {
352 /* Not using a loop because count is fixed and index 1 DNE. */
353 amu_group0_voffset_write(0U, ctx->group0_voffsets[0U]);
354 amu_group0_voffset_write(2U, ctx->group0_voffsets[1U]);
355 amu_group0_voffset_write(3U, ctx->group0_voffsets[2U]);
356 }
357
358 /* Restore group 0 counter configuration */
359 write_amcntenset0_el0(AMU_GROUP0_COUNTERS_MASK);
360
361 #if AMU_GROUP1_NR_COUNTERS
362 /* Restore group 1 counters */
363 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) {
364 if ((AMU_GROUP1_COUNTERS_MASK & (1UL << i)) != 0U) {
365 amu_group1_cnt_write(i, ctx->group1_cnts[i]);
366 }
367 }
368
369 /* Restore group 1 virtual offsets if supported and enabled. */
370 if ((amu_get_version() >= ID_AA64PFR0_AMU_V1P1) &&
371 ((read_hcr_el2() & HCR_AMVOFFEN_BIT) != 0ULL)) {
372 u_register_t amcg1idr = read_amcg1idr_el0() >>
373 AMCG1IDR_VOFF_SHIFT;
374 amcg1idr = amcg1idr & AMU_GROUP1_COUNTERS_MASK;
375
376 for (i = 0U; i < AMU_GROUP1_NR_COUNTERS; i++) {
377 if (((amcg1idr >> i) & 1ULL) != 0ULL) {
378 amu_group1_voffset_write(i,
379 ctx->group1_voffsets[i]);
380 }
381 }
382 }
383
384 /* Restore group 1 counter configuration */
385 write_amcntenset1_el0(AMU_GROUP1_COUNTERS_MASK);
386 #endif
387
388 return (void *)0;
389 }
390
391 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_start, amu_context_save);
392 SUBSCRIBE_TO_EVENT(psci_suspend_pwrdown_finish, amu_context_restore);
393