1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2021 Google LLC
4  * Author: Fuad Tabba <tabba@google.com>
5  */
6 
7 #include <linux/irqchip/arm-gic-v3.h>
8 
9 #include <asm/kvm_asm.h>
10 #include <asm/kvm_mmu.h>
11 
12 #include <hyp/adjust_pc.h>
13 
14 #include <nvhe/fixed_config.h>
15 
16 #include "../../sys_regs.h"
17 
18 /*
19  * Copies of the host's CPU features registers holding sanitized values at hyp.
20  */
21 u64 id_aa64pfr0_el1_sys_val;
22 u64 id_aa64pfr1_el1_sys_val;
23 u64 id_aa64isar0_el1_sys_val;
24 u64 id_aa64isar1_el1_sys_val;
25 u64 id_aa64mmfr0_el1_sys_val;
26 u64 id_aa64mmfr1_el1_sys_val;
27 u64 id_aa64mmfr2_el1_sys_val;
28 
29 /*
30  * Inject an unknown/undefined exception to an AArch64 guest while most of its
31  * sysregs are live.
32  */
inject_undef64(struct kvm_vcpu * vcpu)33 static void inject_undef64(struct kvm_vcpu *vcpu)
34 {
35 	u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
36 
37 	*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
38 	*vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR);
39 
40 	vcpu->arch.flags |= (KVM_ARM64_EXCEPT_AA64_EL1 |
41 			     KVM_ARM64_EXCEPT_AA64_ELx_SYNC |
42 			     KVM_ARM64_PENDING_EXCEPTION);
43 
44 	__kvm_adjust_pc(vcpu);
45 
46 	write_sysreg_el1(esr, SYS_ESR);
47 	write_sysreg_el1(read_sysreg_el2(SYS_ELR), SYS_ELR);
48 	write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
49 	write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR);
50 }
51 
52 /*
53  * Returns the restricted features values of the feature register based on the
54  * limitations in restrict_fields.
55  * A feature id field value of 0b0000 does not impose any restrictions.
56  * Note: Use only for unsigned feature field values.
57  */
get_restricted_features_unsigned(u64 sys_reg_val,u64 restrict_fields)58 static u64 get_restricted_features_unsigned(u64 sys_reg_val,
59 					    u64 restrict_fields)
60 {
61 	u64 value = 0UL;
62 	u64 mask = GENMASK_ULL(ARM64_FEATURE_FIELD_BITS - 1, 0);
63 
64 	/*
65 	 * According to the Arm Architecture Reference Manual, feature fields
66 	 * use increasing values to indicate increases in functionality.
67 	 * Iterate over the restricted feature fields and calculate the minimum
68 	 * unsigned value between the one supported by the system, and what the
69 	 * value is being restricted to.
70 	 */
71 	while (sys_reg_val && restrict_fields) {
72 		value |= min(sys_reg_val & mask, restrict_fields & mask);
73 		sys_reg_val &= ~mask;
74 		restrict_fields &= ~mask;
75 		mask <<= ARM64_FEATURE_FIELD_BITS;
76 	}
77 
78 	return value;
79 }
80 
81 /*
82  * Functions that return the value of feature id registers for protected VMs
83  * based on allowed features, system features, and KVM support.
84  */
85 
get_pvm_id_aa64pfr0(const struct kvm_vcpu * vcpu)86 static u64 get_pvm_id_aa64pfr0(const struct kvm_vcpu *vcpu)
87 {
88 	const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm);
89 	u64 set_mask = 0;
90 	u64 allow_mask = PVM_ID_AA64PFR0_ALLOW;
91 
92 	if (!vcpu_has_sve(vcpu))
93 		allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_SVE);
94 
95 	set_mask |= get_restricted_features_unsigned(id_aa64pfr0_el1_sys_val,
96 		PVM_ID_AA64PFR0_RESTRICT_UNSIGNED);
97 
98 	/* Spectre and Meltdown mitigation in KVM */
99 	set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2),
100 			       (u64)kvm->arch.pfr0_csv2);
101 	set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3),
102 			       (u64)kvm->arch.pfr0_csv3);
103 
104 	return (id_aa64pfr0_el1_sys_val & allow_mask) | set_mask;
105 }
106 
get_pvm_id_aa64pfr1(const struct kvm_vcpu * vcpu)107 static u64 get_pvm_id_aa64pfr1(const struct kvm_vcpu *vcpu)
108 {
109 	const struct kvm *kvm = (const struct kvm *)kern_hyp_va(vcpu->kvm);
110 	u64 allow_mask = PVM_ID_AA64PFR1_ALLOW;
111 
112 	if (!kvm_has_mte(kvm))
113 		allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE);
114 
115 	return id_aa64pfr1_el1_sys_val & allow_mask;
116 }
117 
get_pvm_id_aa64zfr0(const struct kvm_vcpu * vcpu)118 static u64 get_pvm_id_aa64zfr0(const struct kvm_vcpu *vcpu)
119 {
120 	/*
121 	 * No support for Scalable Vectors, therefore, hyp has no sanitized
122 	 * copy of the feature id register.
123 	 */
124 	BUILD_BUG_ON(PVM_ID_AA64ZFR0_ALLOW != 0ULL);
125 	return 0;
126 }
127 
get_pvm_id_aa64dfr0(const struct kvm_vcpu * vcpu)128 static u64 get_pvm_id_aa64dfr0(const struct kvm_vcpu *vcpu)
129 {
130 	/*
131 	 * No support for debug, including breakpoints, and watchpoints,
132 	 * therefore, pKVM has no sanitized copy of the feature id register.
133 	 */
134 	BUILD_BUG_ON(PVM_ID_AA64DFR0_ALLOW != 0ULL);
135 	return 0;
136 }
137 
get_pvm_id_aa64dfr1(const struct kvm_vcpu * vcpu)138 static u64 get_pvm_id_aa64dfr1(const struct kvm_vcpu *vcpu)
139 {
140 	/*
141 	 * No support for debug, therefore, hyp has no sanitized copy of the
142 	 * feature id register.
143 	 */
144 	BUILD_BUG_ON(PVM_ID_AA64DFR1_ALLOW != 0ULL);
145 	return 0;
146 }
147 
get_pvm_id_aa64afr0(const struct kvm_vcpu * vcpu)148 static u64 get_pvm_id_aa64afr0(const struct kvm_vcpu *vcpu)
149 {
150 	/*
151 	 * No support for implementation defined features, therefore, hyp has no
152 	 * sanitized copy of the feature id register.
153 	 */
154 	BUILD_BUG_ON(PVM_ID_AA64AFR0_ALLOW != 0ULL);
155 	return 0;
156 }
157 
get_pvm_id_aa64afr1(const struct kvm_vcpu * vcpu)158 static u64 get_pvm_id_aa64afr1(const struct kvm_vcpu *vcpu)
159 {
160 	/*
161 	 * No support for implementation defined features, therefore, hyp has no
162 	 * sanitized copy of the feature id register.
163 	 */
164 	BUILD_BUG_ON(PVM_ID_AA64AFR1_ALLOW != 0ULL);
165 	return 0;
166 }
167 
get_pvm_id_aa64isar0(const struct kvm_vcpu * vcpu)168 static u64 get_pvm_id_aa64isar0(const struct kvm_vcpu *vcpu)
169 {
170 	return id_aa64isar0_el1_sys_val & PVM_ID_AA64ISAR0_ALLOW;
171 }
172 
get_pvm_id_aa64isar1(const struct kvm_vcpu * vcpu)173 static u64 get_pvm_id_aa64isar1(const struct kvm_vcpu *vcpu)
174 {
175 	u64 allow_mask = PVM_ID_AA64ISAR1_ALLOW;
176 
177 	if (!vcpu_has_ptrauth(vcpu))
178 		allow_mask &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_APA) |
179 				ARM64_FEATURE_MASK(ID_AA64ISAR1_API) |
180 				ARM64_FEATURE_MASK(ID_AA64ISAR1_GPA) |
181 				ARM64_FEATURE_MASK(ID_AA64ISAR1_GPI));
182 
183 	return id_aa64isar1_el1_sys_val & allow_mask;
184 }
185 
get_pvm_id_aa64mmfr0(const struct kvm_vcpu * vcpu)186 static u64 get_pvm_id_aa64mmfr0(const struct kvm_vcpu *vcpu)
187 {
188 	u64 set_mask;
189 
190 	set_mask = get_restricted_features_unsigned(id_aa64mmfr0_el1_sys_val,
191 		PVM_ID_AA64MMFR0_RESTRICT_UNSIGNED);
192 
193 	return (id_aa64mmfr0_el1_sys_val & PVM_ID_AA64MMFR0_ALLOW) | set_mask;
194 }
195 
get_pvm_id_aa64mmfr1(const struct kvm_vcpu * vcpu)196 static u64 get_pvm_id_aa64mmfr1(const struct kvm_vcpu *vcpu)
197 {
198 	return id_aa64mmfr1_el1_sys_val & PVM_ID_AA64MMFR1_ALLOW;
199 }
200 
get_pvm_id_aa64mmfr2(const struct kvm_vcpu * vcpu)201 static u64 get_pvm_id_aa64mmfr2(const struct kvm_vcpu *vcpu)
202 {
203 	return id_aa64mmfr2_el1_sys_val & PVM_ID_AA64MMFR2_ALLOW;
204 }
205 
206 /* Read a sanitized cpufeature ID register by its encoding */
pvm_read_id_reg(const struct kvm_vcpu * vcpu,u32 id)207 u64 pvm_read_id_reg(const struct kvm_vcpu *vcpu, u32 id)
208 {
209 	switch (id) {
210 	case SYS_ID_AA64PFR0_EL1:
211 		return get_pvm_id_aa64pfr0(vcpu);
212 	case SYS_ID_AA64PFR1_EL1:
213 		return get_pvm_id_aa64pfr1(vcpu);
214 	case SYS_ID_AA64ZFR0_EL1:
215 		return get_pvm_id_aa64zfr0(vcpu);
216 	case SYS_ID_AA64DFR0_EL1:
217 		return get_pvm_id_aa64dfr0(vcpu);
218 	case SYS_ID_AA64DFR1_EL1:
219 		return get_pvm_id_aa64dfr1(vcpu);
220 	case SYS_ID_AA64AFR0_EL1:
221 		return get_pvm_id_aa64afr0(vcpu);
222 	case SYS_ID_AA64AFR1_EL1:
223 		return get_pvm_id_aa64afr1(vcpu);
224 	case SYS_ID_AA64ISAR0_EL1:
225 		return get_pvm_id_aa64isar0(vcpu);
226 	case SYS_ID_AA64ISAR1_EL1:
227 		return get_pvm_id_aa64isar1(vcpu);
228 	case SYS_ID_AA64MMFR0_EL1:
229 		return get_pvm_id_aa64mmfr0(vcpu);
230 	case SYS_ID_AA64MMFR1_EL1:
231 		return get_pvm_id_aa64mmfr1(vcpu);
232 	case SYS_ID_AA64MMFR2_EL1:
233 		return get_pvm_id_aa64mmfr2(vcpu);
234 	default:
235 		/*
236 		 * Should never happen because all cases are covered in
237 		 * pvm_sys_reg_descs[].
238 		 */
239 		WARN_ON(1);
240 		break;
241 	}
242 
243 	return 0;
244 }
245 
read_id_reg(const struct kvm_vcpu * vcpu,struct sys_reg_desc const * r)246 static u64 read_id_reg(const struct kvm_vcpu *vcpu,
247 		       struct sys_reg_desc const *r)
248 {
249 	return pvm_read_id_reg(vcpu, reg_to_encoding(r));
250 }
251 
252 /* Handler to RAZ/WI sysregs */
pvm_access_raz_wi(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)253 static bool pvm_access_raz_wi(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
254 			      const struct sys_reg_desc *r)
255 {
256 	if (!p->is_write)
257 		p->regval = 0;
258 
259 	return true;
260 }
261 
262 /*
263  * Accessor for AArch32 feature id registers.
264  *
265  * The value of these registers is "unknown" according to the spec if AArch32
266  * isn't supported.
267  */
pvm_access_id_aarch32(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)268 static bool pvm_access_id_aarch32(struct kvm_vcpu *vcpu,
269 				  struct sys_reg_params *p,
270 				  const struct sys_reg_desc *r)
271 {
272 	if (p->is_write) {
273 		inject_undef64(vcpu);
274 		return false;
275 	}
276 
277 	/*
278 	 * No support for AArch32 guests, therefore, pKVM has no sanitized copy
279 	 * of AArch32 feature id registers.
280 	 */
281 	BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1),
282 		     PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_ELx_64BIT_ONLY);
283 
284 	return pvm_access_raz_wi(vcpu, p, r);
285 }
286 
287 /*
288  * Accessor for AArch64 feature id registers.
289  *
290  * If access is allowed, set the regval to the protected VM's view of the
291  * register and return true.
292  * Otherwise, inject an undefined exception and return false.
293  */
pvm_access_id_aarch64(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)294 static bool pvm_access_id_aarch64(struct kvm_vcpu *vcpu,
295 				  struct sys_reg_params *p,
296 				  const struct sys_reg_desc *r)
297 {
298 	if (p->is_write) {
299 		inject_undef64(vcpu);
300 		return false;
301 	}
302 
303 	p->regval = read_id_reg(vcpu, r);
304 	return true;
305 }
306 
pvm_gic_read_sre(struct kvm_vcpu * vcpu,struct sys_reg_params * p,const struct sys_reg_desc * r)307 static bool pvm_gic_read_sre(struct kvm_vcpu *vcpu,
308 			     struct sys_reg_params *p,
309 			     const struct sys_reg_desc *r)
310 {
311 	/* pVMs only support GICv3. 'nuf said. */
312 	if (!p->is_write)
313 		p->regval = ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB | ICC_SRE_EL1_SRE;
314 
315 	return true;
316 }
317 
318 /* Mark the specified system register as an AArch32 feature id register. */
319 #define AARCH32(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch32 }
320 
321 /* Mark the specified system register as an AArch64 feature id register. */
322 #define AARCH64(REG) { SYS_DESC(REG), .access = pvm_access_id_aarch64 }
323 
324 /* Mark the specified system register as Read-As-Zero/Write-Ignored */
325 #define RAZ_WI(REG) { SYS_DESC(REG), .access = pvm_access_raz_wi }
326 
327 /* Mark the specified system register as not being handled in hyp. */
328 #define HOST_HANDLED(REG) { SYS_DESC(REG), .access = NULL }
329 
330 /*
331  * Architected system registers.
332  * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
333  *
334  * NOTE: Anything not explicitly listed here is *restricted by default*, i.e.,
335  * it will lead to injecting an exception into the guest.
336  */
337 static const struct sys_reg_desc pvm_sys_reg_descs[] = {
338 	/* Cache maintenance by set/way operations are restricted. */
339 
340 	/* Debug and Trace Registers are restricted. */
341 
342 	/* AArch64 mappings of the AArch32 ID registers */
343 	/* CRm=1 */
344 	AARCH32(SYS_ID_PFR0_EL1),
345 	AARCH32(SYS_ID_PFR1_EL1),
346 	AARCH32(SYS_ID_DFR0_EL1),
347 	AARCH32(SYS_ID_AFR0_EL1),
348 	AARCH32(SYS_ID_MMFR0_EL1),
349 	AARCH32(SYS_ID_MMFR1_EL1),
350 	AARCH32(SYS_ID_MMFR2_EL1),
351 	AARCH32(SYS_ID_MMFR3_EL1),
352 
353 	/* CRm=2 */
354 	AARCH32(SYS_ID_ISAR0_EL1),
355 	AARCH32(SYS_ID_ISAR1_EL1),
356 	AARCH32(SYS_ID_ISAR2_EL1),
357 	AARCH32(SYS_ID_ISAR3_EL1),
358 	AARCH32(SYS_ID_ISAR4_EL1),
359 	AARCH32(SYS_ID_ISAR5_EL1),
360 	AARCH32(SYS_ID_MMFR4_EL1),
361 	AARCH32(SYS_ID_ISAR6_EL1),
362 
363 	/* CRm=3 */
364 	AARCH32(SYS_MVFR0_EL1),
365 	AARCH32(SYS_MVFR1_EL1),
366 	AARCH32(SYS_MVFR2_EL1),
367 	AARCH32(SYS_ID_PFR2_EL1),
368 	AARCH32(SYS_ID_DFR1_EL1),
369 	AARCH32(SYS_ID_MMFR5_EL1),
370 
371 	/* AArch64 ID registers */
372 	/* CRm=4 */
373 	AARCH64(SYS_ID_AA64PFR0_EL1),
374 	AARCH64(SYS_ID_AA64PFR1_EL1),
375 	AARCH64(SYS_ID_AA64ZFR0_EL1),
376 	AARCH64(SYS_ID_AA64DFR0_EL1),
377 	AARCH64(SYS_ID_AA64DFR1_EL1),
378 	AARCH64(SYS_ID_AA64AFR0_EL1),
379 	AARCH64(SYS_ID_AA64AFR1_EL1),
380 	AARCH64(SYS_ID_AA64ISAR0_EL1),
381 	AARCH64(SYS_ID_AA64ISAR1_EL1),
382 	AARCH64(SYS_ID_AA64MMFR0_EL1),
383 	AARCH64(SYS_ID_AA64MMFR1_EL1),
384 	AARCH64(SYS_ID_AA64MMFR2_EL1),
385 
386 	/* Scalable Vector Registers are restricted. */
387 
388 	RAZ_WI(SYS_ERRIDR_EL1),
389 	RAZ_WI(SYS_ERRSELR_EL1),
390 	RAZ_WI(SYS_ERXFR_EL1),
391 	RAZ_WI(SYS_ERXCTLR_EL1),
392 	RAZ_WI(SYS_ERXSTATUS_EL1),
393 	RAZ_WI(SYS_ERXADDR_EL1),
394 	RAZ_WI(SYS_ERXMISC0_EL1),
395 	RAZ_WI(SYS_ERXMISC1_EL1),
396 
397 	/* Performance Monitoring Registers are restricted. */
398 
399 	/* Limited Ordering Regions Registers are restricted. */
400 
401 	HOST_HANDLED(SYS_ICC_SGI1R_EL1),
402 	HOST_HANDLED(SYS_ICC_ASGI1R_EL1),
403 	HOST_HANDLED(SYS_ICC_SGI0R_EL1),
404 	{ SYS_DESC(SYS_ICC_SRE_EL1), .access = pvm_gic_read_sre, },
405 
406 	HOST_HANDLED(SYS_CCSIDR_EL1),
407 	HOST_HANDLED(SYS_CLIDR_EL1),
408 	HOST_HANDLED(SYS_CSSELR_EL1),
409 	HOST_HANDLED(SYS_CTR_EL0),
410 
411 	/* Performance Monitoring Registers are restricted. */
412 
413 	/* Activity Monitoring Registers are restricted. */
414 
415 	HOST_HANDLED(SYS_CNTP_TVAL_EL0),
416 	HOST_HANDLED(SYS_CNTP_CTL_EL0),
417 	HOST_HANDLED(SYS_CNTP_CVAL_EL0),
418 
419 	/* Performance Monitoring Registers are restricted. */
420 };
421 
422 /*
423  * Checks that the sysreg table is unique and in-order.
424  *
425  * Returns 0 if the table is consistent, or 1 otherwise.
426  */
kvm_check_pvm_sysreg_table(void)427 int kvm_check_pvm_sysreg_table(void)
428 {
429 	unsigned int i;
430 
431 	for (i = 1; i < ARRAY_SIZE(pvm_sys_reg_descs); i++) {
432 		if (cmp_sys_reg(&pvm_sys_reg_descs[i-1], &pvm_sys_reg_descs[i]) >= 0)
433 			return 1;
434 	}
435 
436 	return 0;
437 }
438 
439 /*
440  * Handler for protected VM MSR, MRS or System instruction execution.
441  *
442  * Returns true if the hypervisor has handled the exit, and control should go
443  * back to the guest, or false if it hasn't, to be handled by the host.
444  */
kvm_handle_pvm_sysreg(struct kvm_vcpu * vcpu,u64 * exit_code)445 bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)
446 {
447 	const struct sys_reg_desc *r;
448 	struct sys_reg_params params;
449 	unsigned long esr = kvm_vcpu_get_esr(vcpu);
450 	int Rt = kvm_vcpu_sys_get_rt(vcpu);
451 
452 	params = esr_sys64_to_params(esr);
453 	params.regval = vcpu_get_reg(vcpu, Rt);
454 
455 	r = find_reg(&params, pvm_sys_reg_descs, ARRAY_SIZE(pvm_sys_reg_descs));
456 
457 	/* Undefined (RESTRICTED). */
458 	if (r == NULL) {
459 		inject_undef64(vcpu);
460 		return true;
461 	}
462 
463 	/* Handled by the host (HOST_HANDLED) */
464 	if (r->access == NULL)
465 		return false;
466 
467 	/* Handled by hyp: skip instruction if instructed to do so. */
468 	if (r->access(vcpu, &params, r))
469 		__kvm_skip_instr(vcpu);
470 
471 	if (!params.is_write)
472 		vcpu_set_reg(vcpu, Rt, params.regval);
473 
474 	return true;
475 }
476 
477 /*
478  * Handler for protected VM restricted exceptions.
479  *
480  * Inject an undefined exception into the guest and return true to indicate that
481  * the hypervisor has handled the exit, and control should go back to the guest.
482  */
kvm_handle_pvm_restricted(struct kvm_vcpu * vcpu,u64 * exit_code)483 bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code)
484 {
485 	inject_undef64(vcpu);
486 	return true;
487 }
488