1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #include <linux/kconfig.h>
24
25 #if IS_REACHABLE(CONFIG_AMD_IOMMU_V2)
26
27 #include <linux/printk.h>
28 #include <linux/device.h>
29 #include <linux/slab.h>
30 #include <linux/pci.h>
31 #include <linux/amd-iommu.h>
32 #include "kfd_priv.h"
33 #include "kfd_dbgmgr.h"
34 #include "kfd_topology.h"
35 #include "kfd_iommu.h"
36
37 static const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
38 AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
39 AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
40
41 /** kfd_iommu_check_device - Check whether IOMMU is available for device
42 */
kfd_iommu_check_device(struct kfd_dev * kfd)43 int kfd_iommu_check_device(struct kfd_dev *kfd)
44 {
45 struct amd_iommu_device_info iommu_info;
46 int err;
47
48 if (!kfd->use_iommu_v2)
49 return -ENODEV;
50
51 iommu_info.flags = 0;
52 err = amd_iommu_device_info(kfd->pdev, &iommu_info);
53 if (err)
54 return err;
55
56 if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags)
57 return -ENODEV;
58
59 return 0;
60 }
61
62 /** kfd_iommu_device_init - Initialize IOMMU for device
63 */
kfd_iommu_device_init(struct kfd_dev * kfd)64 int kfd_iommu_device_init(struct kfd_dev *kfd)
65 {
66 struct amd_iommu_device_info iommu_info;
67 unsigned int pasid_limit;
68 int err;
69
70 if (!kfd->use_iommu_v2)
71 return 0;
72
73 iommu_info.flags = 0;
74 err = amd_iommu_device_info(kfd->pdev, &iommu_info);
75 if (err < 0) {
76 dev_err(kfd_device,
77 "error getting iommu info. is the iommu enabled?\n");
78 return -ENODEV;
79 }
80
81 if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
82 dev_err(kfd_device,
83 "error required iommu flags ats %i, pri %i, pasid %i\n",
84 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
85 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
86 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
87 != 0);
88 return -ENODEV;
89 }
90
91 pasid_limit = min_t(unsigned int,
92 (unsigned int)(1 << kfd->device_info->max_pasid_bits),
93 iommu_info.max_pasids);
94
95 if (!kfd_set_pasid_limit(pasid_limit)) {
96 dev_err(kfd_device, "error setting pasid limit\n");
97 return -EBUSY;
98 }
99
100 return 0;
101 }
102
103 /** kfd_iommu_bind_process_to_device - Have the IOMMU bind a process
104 *
105 * Binds the given process to the given device using its PASID. This
106 * enables IOMMUv2 address translation for the process on the device.
107 *
108 * This function assumes that the process mutex is held.
109 */
kfd_iommu_bind_process_to_device(struct kfd_process_device * pdd)110 int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd)
111 {
112 struct kfd_dev *dev = pdd->dev;
113 struct kfd_process *p = pdd->process;
114 int err;
115
116 if (!dev->use_iommu_v2 || pdd->bound == PDD_BOUND)
117 return 0;
118
119 if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
120 pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
121 return -EINVAL;
122 }
123
124 err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
125 if (!err)
126 pdd->bound = PDD_BOUND;
127
128 return err;
129 }
130
131 /** kfd_iommu_unbind_process - Unbind process from all devices
132 *
133 * This removes all IOMMU device bindings of the process. To be used
134 * before process termination.
135 */
kfd_iommu_unbind_process(struct kfd_process * p)136 void kfd_iommu_unbind_process(struct kfd_process *p)
137 {
138 int i;
139
140 for (i = 0; i < p->n_pdds; i++)
141 if (p->pdds[i]->bound == PDD_BOUND)
142 amd_iommu_unbind_pasid(p->pdds[i]->dev->pdev, p->pasid);
143 }
144
145 /* Callback for process shutdown invoked by the IOMMU driver */
iommu_pasid_shutdown_callback(struct pci_dev * pdev,u32 pasid)146 static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, u32 pasid)
147 {
148 struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
149 struct kfd_process *p;
150 struct kfd_process_device *pdd;
151
152 if (!dev)
153 return;
154
155 /*
156 * Look for the process that matches the pasid. If there is no such
157 * process, we either released it in amdkfd's own notifier, or there
158 * is a bug. Unfortunately, there is no way to tell...
159 */
160 p = kfd_lookup_process_by_pasid(pasid);
161 if (!p)
162 return;
163
164 pr_debug("Unbinding process 0x%x from IOMMU\n", pasid);
165
166 mutex_lock(kfd_get_dbgmgr_mutex());
167
168 if (dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
169 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
170 kfd_dbgmgr_destroy(dev->dbgmgr);
171 dev->dbgmgr = NULL;
172 }
173 }
174
175 mutex_unlock(kfd_get_dbgmgr_mutex());
176
177 mutex_lock(&p->mutex);
178
179 pdd = kfd_get_process_device_data(dev, p);
180 if (pdd)
181 /* For GPU relying on IOMMU, we need to dequeue here
182 * when PASID is still bound.
183 */
184 kfd_process_dequeue_from_device(pdd);
185
186 mutex_unlock(&p->mutex);
187
188 kfd_unref_process(p);
189 }
190
191 /* This function called by IOMMU driver on PPR failure */
iommu_invalid_ppr_cb(struct pci_dev * pdev,u32 pasid,unsigned long address,u16 flags)192 static int iommu_invalid_ppr_cb(struct pci_dev *pdev, u32 pasid,
193 unsigned long address, u16 flags)
194 {
195 struct kfd_dev *dev;
196
197 dev_warn_ratelimited(kfd_device,
198 "Invalid PPR device %x:%x.%x pasid 0x%x address 0x%lX flags 0x%X",
199 pdev->bus->number,
200 PCI_SLOT(pdev->devfn),
201 PCI_FUNC(pdev->devfn),
202 pasid,
203 address,
204 flags);
205
206 dev = kfd_device_by_pci_dev(pdev);
207 if (!WARN_ON(!dev))
208 kfd_signal_iommu_event(dev, pasid, address,
209 flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
210
211 return AMD_IOMMU_INV_PRI_RSP_INVALID;
212 }
213
214 /*
215 * Bind processes do the device that have been temporarily unbound
216 * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
217 */
kfd_bind_processes_to_device(struct kfd_dev * kfd)218 static int kfd_bind_processes_to_device(struct kfd_dev *kfd)
219 {
220 struct kfd_process_device *pdd;
221 struct kfd_process *p;
222 unsigned int temp;
223 int err = 0;
224
225 int idx = srcu_read_lock(&kfd_processes_srcu);
226
227 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
228 mutex_lock(&p->mutex);
229 pdd = kfd_get_process_device_data(kfd, p);
230
231 if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
232 mutex_unlock(&p->mutex);
233 continue;
234 }
235
236 err = amd_iommu_bind_pasid(kfd->pdev, p->pasid,
237 p->lead_thread);
238 if (err < 0) {
239 pr_err("Unexpected pasid 0x%x binding failure\n",
240 p->pasid);
241 mutex_unlock(&p->mutex);
242 break;
243 }
244
245 pdd->bound = PDD_BOUND;
246 mutex_unlock(&p->mutex);
247 }
248
249 srcu_read_unlock(&kfd_processes_srcu, idx);
250
251 return err;
252 }
253
254 /*
255 * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
256 * processes will be restored to PDD_BOUND state in
257 * kfd_bind_processes_to_device.
258 */
kfd_unbind_processes_from_device(struct kfd_dev * kfd)259 static void kfd_unbind_processes_from_device(struct kfd_dev *kfd)
260 {
261 struct kfd_process_device *pdd;
262 struct kfd_process *p;
263 unsigned int temp;
264
265 int idx = srcu_read_lock(&kfd_processes_srcu);
266
267 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
268 mutex_lock(&p->mutex);
269 pdd = kfd_get_process_device_data(kfd, p);
270
271 if (WARN_ON(!pdd)) {
272 mutex_unlock(&p->mutex);
273 continue;
274 }
275
276 if (pdd->bound == PDD_BOUND)
277 pdd->bound = PDD_BOUND_SUSPENDED;
278 mutex_unlock(&p->mutex);
279 }
280
281 srcu_read_unlock(&kfd_processes_srcu, idx);
282 }
283
284 /** kfd_iommu_suspend - Prepare IOMMU for suspend
285 *
286 * This unbinds processes from the device and disables the IOMMU for
287 * the device.
288 */
kfd_iommu_suspend(struct kfd_dev * kfd)289 void kfd_iommu_suspend(struct kfd_dev *kfd)
290 {
291 if (!kfd->use_iommu_v2)
292 return;
293
294 kfd_unbind_processes_from_device(kfd);
295
296 amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
297 amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
298 amd_iommu_free_device(kfd->pdev);
299 }
300
301 /** kfd_iommu_resume - Restore IOMMU after resume
302 *
303 * This reinitializes the IOMMU for the device and re-binds previously
304 * suspended processes to the device.
305 */
kfd_iommu_resume(struct kfd_dev * kfd)306 int kfd_iommu_resume(struct kfd_dev *kfd)
307 {
308 unsigned int pasid_limit;
309 int err;
310
311 if (!kfd->use_iommu_v2)
312 return 0;
313
314 pasid_limit = kfd_get_pasid_limit();
315
316 err = amd_iommu_init_device(kfd->pdev, pasid_limit);
317 if (err)
318 return -ENXIO;
319
320 amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
321 iommu_pasid_shutdown_callback);
322 amd_iommu_set_invalid_ppr_cb(kfd->pdev,
323 iommu_invalid_ppr_cb);
324
325 err = kfd_bind_processes_to_device(kfd);
326 if (err) {
327 amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
328 amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
329 amd_iommu_free_device(kfd->pdev);
330 return err;
331 }
332
333 return 0;
334 }
335
336 /** kfd_iommu_add_perf_counters - Add IOMMU performance counters to topology
337 */
kfd_iommu_add_perf_counters(struct kfd_topology_device * kdev)338 int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev)
339 {
340 struct kfd_perf_properties *props;
341
342 if (!(kdev->node_props.capability & HSA_CAP_ATS_PRESENT))
343 return 0;
344
345 if (!amd_iommu_pc_supported())
346 return 0;
347
348 props = kfd_alloc_struct(props);
349 if (!props)
350 return -ENOMEM;
351 strcpy(props->block_name, "iommu");
352 props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
353 amd_iommu_pc_get_max_counters(0); /* assume one iommu */
354 list_add_tail(&props->list, &kdev->perf_props);
355
356 return 0;
357 }
358
359 #endif
360