1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4 *
5 * Authors:
6 * Anup Patel <anup.patel@wdc.com>
7 */
8
9 #include <linux/bitops.h>
10 #include <linux/cpumask.h>
11 #include <linux/errno.h>
12 #include <linux/err.h>
13 #include <linux/module.h>
14 #include <linux/kvm_host.h>
15 #include <asm/csr.h>
16 #include <asm/sbi.h>
17
18 static unsigned long vmid_version = 1;
19 static unsigned long vmid_next;
20 static unsigned long vmid_bits;
21 static DEFINE_SPINLOCK(vmid_lock);
22
kvm_riscv_stage2_vmid_detect(void)23 void kvm_riscv_stage2_vmid_detect(void)
24 {
25 unsigned long old;
26
27 /* Figure-out number of VMID bits in HW */
28 old = csr_read(CSR_HGATP);
29 csr_write(CSR_HGATP, old | HGATP_VMID_MASK);
30 vmid_bits = csr_read(CSR_HGATP);
31 vmid_bits = (vmid_bits & HGATP_VMID_MASK) >> HGATP_VMID_SHIFT;
32 vmid_bits = fls_long(vmid_bits);
33 csr_write(CSR_HGATP, old);
34
35 /* We polluted local TLB so flush all guest TLB */
36 __kvm_riscv_hfence_gvma_all();
37
38 /* We don't use VMID bits if they are not sufficient */
39 if ((1UL << vmid_bits) < num_possible_cpus())
40 vmid_bits = 0;
41 }
42
kvm_riscv_stage2_vmid_bits(void)43 unsigned long kvm_riscv_stage2_vmid_bits(void)
44 {
45 return vmid_bits;
46 }
47
kvm_riscv_stage2_vmid_init(struct kvm * kvm)48 int kvm_riscv_stage2_vmid_init(struct kvm *kvm)
49 {
50 /* Mark the initial VMID and VMID version invalid */
51 kvm->arch.vmid.vmid_version = 0;
52 kvm->arch.vmid.vmid = 0;
53
54 return 0;
55 }
56
kvm_riscv_stage2_vmid_ver_changed(struct kvm_vmid * vmid)57 bool kvm_riscv_stage2_vmid_ver_changed(struct kvm_vmid *vmid)
58 {
59 if (!vmid_bits)
60 return false;
61
62 return unlikely(READ_ONCE(vmid->vmid_version) !=
63 READ_ONCE(vmid_version));
64 }
65
kvm_riscv_stage2_vmid_update(struct kvm_vcpu * vcpu)66 void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu)
67 {
68 int i;
69 struct kvm_vcpu *v;
70 struct cpumask hmask;
71 struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid;
72
73 if (!kvm_riscv_stage2_vmid_ver_changed(vmid))
74 return;
75
76 spin_lock(&vmid_lock);
77
78 /*
79 * We need to re-check the vmid_version here to ensure that if
80 * another vcpu already allocated a valid vmid for this vm.
81 */
82 if (!kvm_riscv_stage2_vmid_ver_changed(vmid)) {
83 spin_unlock(&vmid_lock);
84 return;
85 }
86
87 /* First user of a new VMID version? */
88 if (unlikely(vmid_next == 0)) {
89 WRITE_ONCE(vmid_version, READ_ONCE(vmid_version) + 1);
90 vmid_next = 1;
91
92 /*
93 * We ran out of VMIDs so we increment vmid_version and
94 * start assigning VMIDs from 1.
95 *
96 * This also means existing VMIDs assignement to all Guest
97 * instances is invalid and we have force VMID re-assignement
98 * for all Guest instances. The Guest instances that were not
99 * running will automatically pick-up new VMIDs because will
100 * call kvm_riscv_stage2_vmid_update() whenever they enter
101 * in-kernel run loop. For Guest instances that are already
102 * running, we force VM exits on all host CPUs using IPI and
103 * flush all Guest TLBs.
104 */
105 riscv_cpuid_to_hartid_mask(cpu_online_mask, &hmask);
106 sbi_remote_hfence_gvma(cpumask_bits(&hmask), 0, 0);
107 }
108
109 vmid->vmid = vmid_next;
110 vmid_next++;
111 vmid_next &= (1 << vmid_bits) - 1;
112
113 WRITE_ONCE(vmid->vmid_version, READ_ONCE(vmid_version));
114
115 spin_unlock(&vmid_lock);
116
117 /* Request stage2 page table update for all VCPUs */
118 kvm_for_each_vcpu(i, v, vcpu->kvm)
119 kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
120 }
121