1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 #ifndef KVM_X86_MMU_SPTE_H
4 #define KVM_X86_MMU_SPTE_H
5 
6 #include "mmu_internal.h"
7 
8 /*
9  * A MMU present SPTE is backed by actual memory and may or may not be present
10  * in hardware.  E.g. MMIO SPTEs are not considered present.  Use bit 11, as it
11  * is ignored by all flavors of SPTEs and checking a low bit often generates
12  * better code than for a high bit, e.g. 56+.  MMU present checks are pervasive
13  * enough that the improved code generation is noticeable in KVM's footprint.
14  */
15 #define SPTE_MMU_PRESENT_MASK		BIT_ULL(11)
16 
17 /*
18  * TDP SPTES (more specifically, EPT SPTEs) may not have A/D bits, and may also
19  * be restricted to using write-protection (for L2 when CPU dirty logging, i.e.
20  * PML, is enabled).  Use bits 52 and 53 to hold the type of A/D tracking that
21  * is must be employed for a given TDP SPTE.
22  *
23  * Note, the "enabled" mask must be '0', as bits 62:52 are _reserved_ for PAE
24  * paging, including NPT PAE.  This scheme works because legacy shadow paging
25  * is guaranteed to have A/D bits and write-protection is forced only for
26  * TDP with CPU dirty logging (PML).  If NPT ever gains PML-like support, it
27  * must be restricted to 64-bit KVM.
28  */
29 #define SPTE_TDP_AD_SHIFT		52
30 #define SPTE_TDP_AD_MASK		(3ULL << SPTE_TDP_AD_SHIFT)
31 #define SPTE_TDP_AD_ENABLED_MASK	(0ULL << SPTE_TDP_AD_SHIFT)
32 #define SPTE_TDP_AD_DISABLED_MASK	(1ULL << SPTE_TDP_AD_SHIFT)
33 #define SPTE_TDP_AD_WRPROT_ONLY_MASK	(2ULL << SPTE_TDP_AD_SHIFT)
34 static_assert(SPTE_TDP_AD_ENABLED_MASK == 0);
35 
36 #ifdef CONFIG_DYNAMIC_PHYSICAL_MASK
37 #define PT64_BASE_ADDR_MASK (physical_mask & ~(u64)(PAGE_SIZE-1))
38 #else
39 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
40 #endif
41 
42 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
43 			| shadow_x_mask | shadow_nx_mask | shadow_me_mask)
44 
45 #define ACC_EXEC_MASK    1
46 #define ACC_WRITE_MASK   PT_WRITABLE_MASK
47 #define ACC_USER_MASK    PT_USER_MASK
48 #define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
49 
50 /* The mask for the R/X bits in EPT PTEs */
51 #define PT64_EPT_READABLE_MASK			0x1ull
52 #define PT64_EPT_EXECUTABLE_MASK		0x4ull
53 
54 #define PT64_LEVEL_BITS 9
55 
56 #define PT64_LEVEL_SHIFT(level) \
57 		(PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
58 
59 #define PT64_INDEX(address, level)\
60 	(((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
61 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
62 
63 /* Bits 9 and 10 are ignored by all non-EPT PTEs. */
64 #define DEFAULT_SPTE_HOST_WRITEABLE	BIT_ULL(9)
65 #define DEFAULT_SPTE_MMU_WRITEABLE	BIT_ULL(10)
66 
67 /*
68  * The mask/shift to use for saving the original R/X bits when marking the PTE
69  * as not-present for access tracking purposes. We do not save the W bit as the
70  * PTEs being access tracked also need to be dirty tracked, so the W bit will be
71  * restored only when a write is attempted to the page.  This mask obviously
72  * must not overlap the A/D type mask.
73  */
74 #define SHADOW_ACC_TRACK_SAVED_BITS_MASK (PT64_EPT_READABLE_MASK | \
75 					  PT64_EPT_EXECUTABLE_MASK)
76 #define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT 54
77 #define SHADOW_ACC_TRACK_SAVED_MASK	(SHADOW_ACC_TRACK_SAVED_BITS_MASK << \
78 					 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
79 static_assert(!(SPTE_TDP_AD_MASK & SHADOW_ACC_TRACK_SAVED_MASK));
80 
81 /*
82  * Low ignored bits are at a premium for EPT, use high ignored bits, taking care
83  * to not overlap the A/D type mask or the saved access bits of access-tracked
84  * SPTEs when A/D bits are disabled.
85  */
86 #define EPT_SPTE_HOST_WRITABLE		BIT_ULL(57)
87 #define EPT_SPTE_MMU_WRITABLE		BIT_ULL(58)
88 
89 static_assert(!(EPT_SPTE_HOST_WRITABLE & SPTE_TDP_AD_MASK));
90 static_assert(!(EPT_SPTE_MMU_WRITABLE & SPTE_TDP_AD_MASK));
91 static_assert(!(EPT_SPTE_HOST_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
92 static_assert(!(EPT_SPTE_MMU_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
93 
94 /* Defined only to keep the above static asserts readable. */
95 #undef SHADOW_ACC_TRACK_SAVED_MASK
96 
97 /*
98  * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of
99  * the memslots generation and is derived as follows:
100  *
101  * Bits 0-7 of the MMIO generation are propagated to spte bits 3-10
102  * Bits 8-18 of the MMIO generation are propagated to spte bits 52-62
103  *
104  * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in
105  * the MMIO generation number, as doing so would require stealing a bit from
106  * the "real" generation number and thus effectively halve the maximum number
107  * of MMIO generations that can be handled before encountering a wrap (which
108  * requires a full MMU zap).  The flag is instead explicitly queried when
109  * checking for MMIO spte cache hits.
110  */
111 
112 #define MMIO_SPTE_GEN_LOW_START		3
113 #define MMIO_SPTE_GEN_LOW_END		10
114 
115 #define MMIO_SPTE_GEN_HIGH_START	52
116 #define MMIO_SPTE_GEN_HIGH_END		62
117 
118 #define MMIO_SPTE_GEN_LOW_MASK		GENMASK_ULL(MMIO_SPTE_GEN_LOW_END, \
119 						    MMIO_SPTE_GEN_LOW_START)
120 #define MMIO_SPTE_GEN_HIGH_MASK		GENMASK_ULL(MMIO_SPTE_GEN_HIGH_END, \
121 						    MMIO_SPTE_GEN_HIGH_START)
122 static_assert(!(SPTE_MMU_PRESENT_MASK &
123 		(MMIO_SPTE_GEN_LOW_MASK | MMIO_SPTE_GEN_HIGH_MASK)));
124 
125 #define MMIO_SPTE_GEN_LOW_BITS		(MMIO_SPTE_GEN_LOW_END - MMIO_SPTE_GEN_LOW_START + 1)
126 #define MMIO_SPTE_GEN_HIGH_BITS		(MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1)
127 
128 /* remember to adjust the comment above as well if you change these */
129 static_assert(MMIO_SPTE_GEN_LOW_BITS == 8 && MMIO_SPTE_GEN_HIGH_BITS == 11);
130 
131 #define MMIO_SPTE_GEN_LOW_SHIFT		(MMIO_SPTE_GEN_LOW_START - 0)
132 #define MMIO_SPTE_GEN_HIGH_SHIFT	(MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS)
133 
134 #define MMIO_SPTE_GEN_MASK		GENMASK_ULL(MMIO_SPTE_GEN_LOW_BITS + MMIO_SPTE_GEN_HIGH_BITS - 1, 0)
135 
136 extern u64 __read_mostly shadow_host_writable_mask;
137 extern u64 __read_mostly shadow_mmu_writable_mask;
138 extern u64 __read_mostly shadow_nx_mask;
139 extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
140 extern u64 __read_mostly shadow_user_mask;
141 extern u64 __read_mostly shadow_accessed_mask;
142 extern u64 __read_mostly shadow_dirty_mask;
143 extern u64 __read_mostly shadow_mmio_value;
144 extern u64 __read_mostly shadow_mmio_mask;
145 extern u64 __read_mostly shadow_mmio_access_mask;
146 extern u64 __read_mostly shadow_present_mask;
147 extern u64 __read_mostly shadow_me_mask;
148 
149 /*
150  * SPTEs in MMUs without A/D bits are marked with SPTE_TDP_AD_DISABLED_MASK;
151  * shadow_acc_track_mask is the set of bits to be cleared in non-accessed
152  * pages.
153  */
154 extern u64 __read_mostly shadow_acc_track_mask;
155 
156 /*
157  * This mask must be set on all non-zero Non-Present or Reserved SPTEs in order
158  * to guard against L1TF attacks.
159  */
160 extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask;
161 
162 /*
163  * The number of high-order 1 bits to use in the mask above.
164  */
165 #define SHADOW_NONPRESENT_OR_RSVD_MASK_LEN 5
166 
167 /*
168  * If a thread running without exclusive control of the MMU lock must perform a
169  * multi-part operation on an SPTE, it can set the SPTE to REMOVED_SPTE as a
170  * non-present intermediate value. Other threads which encounter this value
171  * should not modify the SPTE.
172  *
173  * Use a semi-arbitrary value that doesn't set RWX bits, i.e. is not-present on
174  * bot AMD and Intel CPUs, and doesn't set PFN bits, i.e. doesn't create a L1TF
175  * vulnerability.  Use only low bits to avoid 64-bit immediates.
176  *
177  * Only used by the TDP MMU.
178  */
179 #define REMOVED_SPTE	0x5a0ULL
180 
181 /* Removed SPTEs must not be misconstrued as shadow present PTEs. */
182 static_assert(!(REMOVED_SPTE & SPTE_MMU_PRESENT_MASK));
183 
is_removed_spte(u64 spte)184 static inline bool is_removed_spte(u64 spte)
185 {
186 	return spte == REMOVED_SPTE;
187 }
188 
189 /*
190  * In some cases, we need to preserve the GFN of a non-present or reserved
191  * SPTE when we usurp the upper five bits of the physical address space to
192  * defend against L1TF, e.g. for MMIO SPTEs.  To preserve the GFN, we'll
193  * shift bits of the GFN that overlap with shadow_nonpresent_or_rsvd_mask
194  * left into the reserved bits, i.e. the GFN in the SPTE will be split into
195  * high and low parts.  This mask covers the lower bits of the GFN.
196  */
197 extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask;
198 
199 /*
200  * The number of non-reserved physical address bits irrespective of features
201  * that repurpose legal bits, e.g. MKTME.
202  */
203 extern u8 __read_mostly shadow_phys_bits;
204 
is_mmio_spte(u64 spte)205 static inline bool is_mmio_spte(u64 spte)
206 {
207 	return (spte & shadow_mmio_mask) == shadow_mmio_value &&
208 	       likely(shadow_mmio_value);
209 }
210 
is_shadow_present_pte(u64 pte)211 static inline bool is_shadow_present_pte(u64 pte)
212 {
213 	return !!(pte & SPTE_MMU_PRESENT_MASK);
214 }
215 
sp_ad_disabled(struct kvm_mmu_page * sp)216 static inline bool sp_ad_disabled(struct kvm_mmu_page *sp)
217 {
218 	return sp->role.ad_disabled;
219 }
220 
spte_ad_enabled(u64 spte)221 static inline bool spte_ad_enabled(u64 spte)
222 {
223 	MMU_WARN_ON(!is_shadow_present_pte(spte));
224 	return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_DISABLED_MASK;
225 }
226 
spte_ad_need_write_protect(u64 spte)227 static inline bool spte_ad_need_write_protect(u64 spte)
228 {
229 	MMU_WARN_ON(!is_shadow_present_pte(spte));
230 	/*
231 	 * This is benign for non-TDP SPTEs as SPTE_TDP_AD_ENABLED_MASK is '0',
232 	 * and non-TDP SPTEs will never set these bits.  Optimize for 64-bit
233 	 * TDP and do the A/D type check unconditionally.
234 	 */
235 	return (spte & SPTE_TDP_AD_MASK) != SPTE_TDP_AD_ENABLED_MASK;
236 }
237 
spte_shadow_accessed_mask(u64 spte)238 static inline u64 spte_shadow_accessed_mask(u64 spte)
239 {
240 	MMU_WARN_ON(!is_shadow_present_pte(spte));
241 	return spte_ad_enabled(spte) ? shadow_accessed_mask : 0;
242 }
243 
spte_shadow_dirty_mask(u64 spte)244 static inline u64 spte_shadow_dirty_mask(u64 spte)
245 {
246 	MMU_WARN_ON(!is_shadow_present_pte(spte));
247 	return spte_ad_enabled(spte) ? shadow_dirty_mask : 0;
248 }
249 
is_access_track_spte(u64 spte)250 static inline bool is_access_track_spte(u64 spte)
251 {
252 	return !spte_ad_enabled(spte) && (spte & shadow_acc_track_mask) == 0;
253 }
254 
is_large_pte(u64 pte)255 static inline bool is_large_pte(u64 pte)
256 {
257 	return pte & PT_PAGE_SIZE_MASK;
258 }
259 
is_last_spte(u64 pte,int level)260 static inline bool is_last_spte(u64 pte, int level)
261 {
262 	return (level == PG_LEVEL_4K) || is_large_pte(pte);
263 }
264 
is_executable_pte(u64 spte)265 static inline bool is_executable_pte(u64 spte)
266 {
267 	return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
268 }
269 
spte_to_pfn(u64 pte)270 static inline kvm_pfn_t spte_to_pfn(u64 pte)
271 {
272 	return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
273 }
274 
is_accessed_spte(u64 spte)275 static inline bool is_accessed_spte(u64 spte)
276 {
277 	u64 accessed_mask = spte_shadow_accessed_mask(spte);
278 
279 	return accessed_mask ? spte & accessed_mask
280 			     : !is_access_track_spte(spte);
281 }
282 
is_dirty_spte(u64 spte)283 static inline bool is_dirty_spte(u64 spte)
284 {
285 	u64 dirty_mask = spte_shadow_dirty_mask(spte);
286 
287 	return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK;
288 }
289 
get_rsvd_bits(struct rsvd_bits_validate * rsvd_check,u64 pte,int level)290 static inline u64 get_rsvd_bits(struct rsvd_bits_validate *rsvd_check, u64 pte,
291 				int level)
292 {
293 	int bit7 = (pte >> 7) & 1;
294 
295 	return rsvd_check->rsvd_bits_mask[bit7][level-1];
296 }
297 
__is_rsvd_bits_set(struct rsvd_bits_validate * rsvd_check,u64 pte,int level)298 static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check,
299 				      u64 pte, int level)
300 {
301 	return pte & get_rsvd_bits(rsvd_check, pte, level);
302 }
303 
__is_bad_mt_xwr(struct rsvd_bits_validate * rsvd_check,u64 pte)304 static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check,
305 				   u64 pte)
306 {
307 	return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f);
308 }
309 
is_rsvd_spte(struct rsvd_bits_validate * rsvd_check,u64 spte,int level)310 static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check,
311 					 u64 spte, int level)
312 {
313 	return __is_bad_mt_xwr(rsvd_check, spte) ||
314 	       __is_rsvd_bits_set(rsvd_check, spte, level);
315 }
316 
spte_can_locklessly_be_made_writable(u64 spte)317 static inline bool spte_can_locklessly_be_made_writable(u64 spte)
318 {
319 	return (spte & shadow_host_writable_mask) &&
320 	       (spte & shadow_mmu_writable_mask);
321 }
322 
get_mmio_spte_generation(u64 spte)323 static inline u64 get_mmio_spte_generation(u64 spte)
324 {
325 	u64 gen;
326 
327 	gen = (spte & MMIO_SPTE_GEN_LOW_MASK) >> MMIO_SPTE_GEN_LOW_SHIFT;
328 	gen |= (spte & MMIO_SPTE_GEN_HIGH_MASK) >> MMIO_SPTE_GEN_HIGH_SHIFT;
329 	return gen;
330 }
331 
332 bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
333 	       struct kvm_memory_slot *slot,
334 	       unsigned int pte_access, gfn_t gfn, kvm_pfn_t pfn,
335 	       u64 old_spte, bool prefetch, bool can_unsync,
336 	       bool host_writable, u64 *new_spte);
337 u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled);
338 u64 make_mmio_spte(struct kvm_vcpu *vcpu, u64 gfn, unsigned int access);
339 u64 mark_spte_for_access_track(u64 spte);
340 u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn);
341 
342 void kvm_mmu_reset_all_pte_masks(void);
343 
344 #endif
345