1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __KVM_X86_MMU_H
3 #define __KVM_X86_MMU_H
4 
5 #include <linux/kvm_host.h>
6 #include "kvm_cache_regs.h"
7 #include "cpuid.h"
8 
9 #define PT64_PT_BITS 9
10 #define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
11 #define PT32_PT_BITS 10
12 #define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
13 
14 #define PT_WRITABLE_SHIFT 1
15 #define PT_USER_SHIFT 2
16 
17 #define PT_PRESENT_MASK (1ULL << 0)
18 #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
19 #define PT_USER_MASK (1ULL << PT_USER_SHIFT)
20 #define PT_PWT_MASK (1ULL << 3)
21 #define PT_PCD_MASK (1ULL << 4)
22 #define PT_ACCESSED_SHIFT 5
23 #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
24 #define PT_DIRTY_SHIFT 6
25 #define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
26 #define PT_PAGE_SIZE_SHIFT 7
27 #define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT)
28 #define PT_PAT_MASK (1ULL << 7)
29 #define PT_GLOBAL_MASK (1ULL << 8)
30 #define PT64_NX_SHIFT 63
31 #define PT64_NX_MASK (1ULL << PT64_NX_SHIFT)
32 
33 #define PT_PAT_SHIFT 7
34 #define PT_DIR_PAT_SHIFT 12
35 #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
36 
37 #define PT32_DIR_PSE36_SIZE 4
38 #define PT32_DIR_PSE36_SHIFT 13
39 #define PT32_DIR_PSE36_MASK \
40 	(((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
41 
42 #define PT64_ROOT_5LEVEL 5
43 #define PT64_ROOT_4LEVEL 4
44 #define PT32_ROOT_LEVEL 2
45 #define PT32E_ROOT_LEVEL 3
46 
47 #define KVM_MMU_CR4_ROLE_BITS (X86_CR4_PSE | X86_CR4_PAE | X86_CR4_LA57 | \
48 			       X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE)
49 
50 #define KVM_MMU_CR0_ROLE_BITS (X86_CR0_PG | X86_CR0_WP)
51 
rsvd_bits(int s,int e)52 static __always_inline u64 rsvd_bits(int s, int e)
53 {
54 	BUILD_BUG_ON(__builtin_constant_p(e) && __builtin_constant_p(s) && e < s);
55 
56 	if (__builtin_constant_p(e))
57 		BUILD_BUG_ON(e > 63);
58 	else
59 		e &= 63;
60 
61 	if (e < s)
62 		return 0;
63 
64 	return ((2ULL << (e - s)) - 1) << s;
65 }
66 
67 void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask);
68 void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only);
69 
70 void kvm_init_mmu(struct kvm_vcpu *vcpu);
71 void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
72 			     unsigned long cr4, u64 efer, gpa_t nested_cr3);
73 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
74 			     bool accessed_dirty, gpa_t new_eptp);
75 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu);
76 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
77 				u64 fault_address, char *insn, int insn_len);
78 
79 int kvm_mmu_load(struct kvm_vcpu *vcpu);
80 void kvm_mmu_unload(struct kvm_vcpu *vcpu);
81 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
82 void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu);
83 
kvm_mmu_reload(struct kvm_vcpu * vcpu)84 static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu)
85 {
86 	if (likely(vcpu->arch.mmu->root_hpa != INVALID_PAGE))
87 		return 0;
88 
89 	return kvm_mmu_load(vcpu);
90 }
91 
kvm_get_pcid(struct kvm_vcpu * vcpu,gpa_t cr3)92 static inline unsigned long kvm_get_pcid(struct kvm_vcpu *vcpu, gpa_t cr3)
93 {
94 	BUILD_BUG_ON((X86_CR3_PCID_MASK & PAGE_MASK) != 0);
95 
96 	return kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)
97 	       ? cr3 & X86_CR3_PCID_MASK
98 	       : 0;
99 }
100 
kvm_get_active_pcid(struct kvm_vcpu * vcpu)101 static inline unsigned long kvm_get_active_pcid(struct kvm_vcpu *vcpu)
102 {
103 	return kvm_get_pcid(vcpu, kvm_read_cr3(vcpu));
104 }
105 
kvm_mmu_load_pgd(struct kvm_vcpu * vcpu)106 static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)
107 {
108 	u64 root_hpa = vcpu->arch.mmu->root_hpa;
109 
110 	if (!VALID_PAGE(root_hpa))
111 		return;
112 
113 	static_call(kvm_x86_load_mmu_pgd)(vcpu, root_hpa,
114 					  vcpu->arch.mmu->shadow_root_level);
115 }
116 
117 struct kvm_page_fault {
118 	/* arguments to kvm_mmu_do_page_fault.  */
119 	const gpa_t addr;
120 	const u32 error_code;
121 	const bool prefetch;
122 
123 	/* Derived from error_code.  */
124 	const bool exec;
125 	const bool write;
126 	const bool present;
127 	const bool rsvd;
128 	const bool user;
129 
130 	/* Derived from mmu and global state.  */
131 	const bool is_tdp;
132 	const bool nx_huge_page_workaround_enabled;
133 
134 	/*
135 	 * Whether a >4KB mapping can be created or is forbidden due to NX
136 	 * hugepages.
137 	 */
138 	bool huge_page_disallowed;
139 
140 	/*
141 	 * Maximum page size that can be created for this fault; input to
142 	 * FNAME(fetch), __direct_map and kvm_tdp_mmu_map.
143 	 */
144 	u8 max_level;
145 
146 	/*
147 	 * Page size that can be created based on the max_level and the
148 	 * page size used by the host mapping.
149 	 */
150 	u8 req_level;
151 
152 	/*
153 	 * Page size that will be created based on the req_level and
154 	 * huge_page_disallowed.
155 	 */
156 	u8 goal_level;
157 
158 	/* Shifted addr, or result of guest page table walk if addr is a gva.  */
159 	gfn_t gfn;
160 
161 	/* The memslot containing gfn. May be NULL. */
162 	struct kvm_memory_slot *slot;
163 
164 	/* Outputs of kvm_faultin_pfn.  */
165 	kvm_pfn_t pfn;
166 	hva_t hva;
167 	bool map_writable;
168 };
169 
170 int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
171 
172 extern int nx_huge_pages;
is_nx_huge_page_enabled(void)173 static inline bool is_nx_huge_page_enabled(void)
174 {
175 	return READ_ONCE(nx_huge_pages);
176 }
177 
kvm_mmu_do_page_fault(struct kvm_vcpu * vcpu,gpa_t cr2_or_gpa,u32 err,bool prefetch)178 static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
179 					u32 err, bool prefetch)
180 {
181 	struct kvm_page_fault fault = {
182 		.addr = cr2_or_gpa,
183 		.error_code = err,
184 		.exec = err & PFERR_FETCH_MASK,
185 		.write = err & PFERR_WRITE_MASK,
186 		.present = err & PFERR_PRESENT_MASK,
187 		.rsvd = err & PFERR_RSVD_MASK,
188 		.user = err & PFERR_USER_MASK,
189 		.prefetch = prefetch,
190 		.is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault),
191 		.nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(),
192 
193 		.max_level = KVM_MAX_HUGEPAGE_LEVEL,
194 		.req_level = PG_LEVEL_4K,
195 		.goal_level = PG_LEVEL_4K,
196 	};
197 #ifdef CONFIG_RETPOLINE
198 	if (fault.is_tdp)
199 		return kvm_tdp_page_fault(vcpu, &fault);
200 #endif
201 	return vcpu->arch.mmu->page_fault(vcpu, &fault);
202 }
203 
204 /*
205  * Currently, we have two sorts of write-protection, a) the first one
206  * write-protects guest page to sync the guest modification, b) another one is
207  * used to sync dirty bitmap when we do KVM_GET_DIRTY_LOG. The differences
208  * between these two sorts are:
209  * 1) the first case clears MMU-writable bit.
210  * 2) the first case requires flushing tlb immediately avoiding corrupting
211  *    shadow page table between all vcpus so it should be in the protection of
212  *    mmu-lock. And the another case does not need to flush tlb until returning
213  *    the dirty bitmap to userspace since it only write-protects the page
214  *    logged in the bitmap, that means the page in the dirty bitmap is not
215  *    missed, so it can flush tlb out of mmu-lock.
216  *
217  * So, there is the problem: the first case can meet the corrupted tlb caused
218  * by another case which write-protects pages but without flush tlb
219  * immediately. In order to making the first case be aware this problem we let
220  * it flush tlb if we try to write-protect a spte whose MMU-writable bit
221  * is set, it works since another case never touches MMU-writable bit.
222  *
223  * Anyway, whenever a spte is updated (only permission and status bits are
224  * changed) we need to check whether the spte with MMU-writable becomes
225  * readonly, if that happens, we need to flush tlb. Fortunately,
226  * mmu_spte_update() has already handled it perfectly.
227  *
228  * The rules to use MMU-writable and PT_WRITABLE_MASK:
229  * - if we want to see if it has writable tlb entry or if the spte can be
230  *   writable on the mmu mapping, check MMU-writable, this is the most
231  *   case, otherwise
232  * - if we fix page fault on the spte or do write-protection by dirty logging,
233  *   check PT_WRITABLE_MASK.
234  *
235  * TODO: introduce APIs to split these two cases.
236  */
is_writable_pte(unsigned long pte)237 static inline bool is_writable_pte(unsigned long pte)
238 {
239 	return pte & PT_WRITABLE_MASK;
240 }
241 
242 /*
243  * Check if a given access (described through the I/D, W/R and U/S bits of a
244  * page fault error code pfec) causes a permission fault with the given PTE
245  * access rights (in ACC_* format).
246  *
247  * Return zero if the access does not fault; return the page fault error code
248  * if the access faults.
249  */
permission_fault(struct kvm_vcpu * vcpu,struct kvm_mmu * mmu,unsigned pte_access,unsigned pte_pkey,unsigned pfec)250 static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
251 				  unsigned pte_access, unsigned pte_pkey,
252 				  unsigned pfec)
253 {
254 	int cpl = static_call(kvm_x86_get_cpl)(vcpu);
255 	unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
256 
257 	/*
258 	 * If CPL < 3, SMAP prevention are disabled if EFLAGS.AC = 1.
259 	 *
260 	 * If CPL = 3, SMAP applies to all supervisor-mode data accesses
261 	 * (these are implicit supervisor accesses) regardless of the value
262 	 * of EFLAGS.AC.
263 	 *
264 	 * This computes (cpl < 3) && (rflags & X86_EFLAGS_AC), leaving
265 	 * the result in X86_EFLAGS_AC. We then insert it in place of
266 	 * the PFERR_RSVD_MASK bit; this bit will always be zero in pfec,
267 	 * but it will be one in index if SMAP checks are being overridden.
268 	 * It is important to keep this branchless.
269 	 */
270 	unsigned long smap = (cpl - 3) & (rflags & X86_EFLAGS_AC);
271 	int index = (pfec >> 1) +
272 		    (smap >> (X86_EFLAGS_AC_BIT - PFERR_RSVD_BIT + 1));
273 	bool fault = (mmu->permissions[index] >> pte_access) & 1;
274 	u32 errcode = PFERR_PRESENT_MASK;
275 
276 	WARN_ON(pfec & (PFERR_PK_MASK | PFERR_RSVD_MASK));
277 	if (unlikely(mmu->pkru_mask)) {
278 		u32 pkru_bits, offset;
279 
280 		/*
281 		* PKRU defines 32 bits, there are 16 domains and 2
282 		* attribute bits per domain in pkru.  pte_pkey is the
283 		* index of the protection domain, so pte_pkey * 2 is
284 		* is the index of the first bit for the domain.
285 		*/
286 		pkru_bits = (vcpu->arch.pkru >> (pte_pkey * 2)) & 3;
287 
288 		/* clear present bit, replace PFEC.RSVD with ACC_USER_MASK. */
289 		offset = (pfec & ~1) +
290 			((pte_access & PT_USER_MASK) << (PFERR_RSVD_BIT - PT_USER_SHIFT));
291 
292 		pkru_bits &= mmu->pkru_mask >> offset;
293 		errcode |= -pkru_bits & PFERR_PK_MASK;
294 		fault |= (pkru_bits != 0);
295 	}
296 
297 	return -(u32)fault & errcode;
298 }
299 
300 void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);
301 
302 int kvm_arch_write_log_dirty(struct kvm_vcpu *vcpu);
303 
304 int kvm_mmu_post_init_vm(struct kvm *kvm);
305 void kvm_mmu_pre_destroy_vm(struct kvm *kvm);
306 
kvm_shadow_root_allocated(struct kvm * kvm)307 static inline bool kvm_shadow_root_allocated(struct kvm *kvm)
308 {
309 	/*
310 	 * Read shadow_root_allocated before related pointers. Hence, threads
311 	 * reading shadow_root_allocated in any lock context are guaranteed to
312 	 * see the pointers. Pairs with smp_store_release in
313 	 * mmu_first_shadow_root_alloc.
314 	 */
315 	return smp_load_acquire(&kvm->arch.shadow_root_allocated);
316 }
317 
318 #ifdef CONFIG_X86_64
is_tdp_mmu_enabled(struct kvm * kvm)319 static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return kvm->arch.tdp_mmu_enabled; }
320 #else
is_tdp_mmu_enabled(struct kvm * kvm)321 static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return false; }
322 #endif
323 
kvm_memslots_have_rmaps(struct kvm * kvm)324 static inline bool kvm_memslots_have_rmaps(struct kvm *kvm)
325 {
326 	return !is_tdp_mmu_enabled(kvm) || kvm_shadow_root_allocated(kvm);
327 }
328 
gfn_to_index(gfn_t gfn,gfn_t base_gfn,int level)329 static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level)
330 {
331 	/* KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K) must be 0. */
332 	return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) -
333 		(base_gfn >> KVM_HPAGE_GFN_SHIFT(level));
334 }
335 
336 static inline unsigned long
__kvm_mmu_slot_lpages(struct kvm_memory_slot * slot,unsigned long npages,int level)337 __kvm_mmu_slot_lpages(struct kvm_memory_slot *slot, unsigned long npages,
338 		      int level)
339 {
340 	return gfn_to_index(slot->base_gfn + npages - 1,
341 			    slot->base_gfn, level) + 1;
342 }
343 
344 static inline unsigned long
kvm_mmu_slot_lpages(struct kvm_memory_slot * slot,int level)345 kvm_mmu_slot_lpages(struct kvm_memory_slot *slot, int level)
346 {
347 	return __kvm_mmu_slot_lpages(slot, slot->npages, level);
348 }
349 
kvm_update_page_stats(struct kvm * kvm,int level,int count)350 static inline void kvm_update_page_stats(struct kvm *kvm, int level, int count)
351 {
352 	atomic64_add(count, &kvm->stat.pages[level - 1]);
353 }
354 #endif
355