1 /*
2 * Copyright 2020 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25 #include <linux/dma-buf-map.h>
26 #include <linux/io-mapping.h>
27 #include <linux/scatterlist.h>
28
29 #include <drm/ttm/ttm_resource.h>
30 #include <drm/ttm/ttm_bo_driver.h>
31
ttm_resource_init(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource * res)32 void ttm_resource_init(struct ttm_buffer_object *bo,
33 const struct ttm_place *place,
34 struct ttm_resource *res)
35 {
36 res->start = 0;
37 res->num_pages = PFN_UP(bo->base.size);
38 res->mem_type = place->mem_type;
39 res->placement = place->flags;
40 res->bus.addr = NULL;
41 res->bus.offset = 0;
42 res->bus.is_iomem = false;
43 res->bus.caching = ttm_cached;
44 }
45 EXPORT_SYMBOL(ttm_resource_init);
46
ttm_resource_alloc(struct ttm_buffer_object * bo,const struct ttm_place * place,struct ttm_resource ** res_ptr)47 int ttm_resource_alloc(struct ttm_buffer_object *bo,
48 const struct ttm_place *place,
49 struct ttm_resource **res_ptr)
50 {
51 struct ttm_resource_manager *man =
52 ttm_manager_type(bo->bdev, place->mem_type);
53
54 return man->func->alloc(man, bo, place, res_ptr);
55 }
56
ttm_resource_free(struct ttm_buffer_object * bo,struct ttm_resource ** res)57 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
58 {
59 struct ttm_resource_manager *man;
60
61 if (!*res)
62 return;
63
64 man = ttm_manager_type(bo->bdev, (*res)->mem_type);
65 man->func->free(man, *res);
66 *res = NULL;
67 }
68 EXPORT_SYMBOL(ttm_resource_free);
69
ttm_resource_places_compat(struct ttm_resource * res,const struct ttm_place * places,unsigned num_placement)70 static bool ttm_resource_places_compat(struct ttm_resource *res,
71 const struct ttm_place *places,
72 unsigned num_placement)
73 {
74 unsigned i;
75
76 if (res->placement & TTM_PL_FLAG_TEMPORARY)
77 return false;
78
79 for (i = 0; i < num_placement; i++) {
80 const struct ttm_place *heap = &places[i];
81
82 if (res->start < heap->fpfn || (heap->lpfn &&
83 (res->start + res->num_pages) > heap->lpfn))
84 continue;
85
86 if ((res->mem_type == heap->mem_type) &&
87 (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
88 (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
89 return true;
90 }
91 return false;
92 }
93
94 /**
95 * ttm_resource_compat - check if resource is compatible with placement
96 *
97 * @res: the resource to check
98 * @placement: the placement to check against
99 *
100 * Returns true if the placement is compatible.
101 */
ttm_resource_compat(struct ttm_resource * res,struct ttm_placement * placement)102 bool ttm_resource_compat(struct ttm_resource *res,
103 struct ttm_placement *placement)
104 {
105 if (ttm_resource_places_compat(res, placement->placement,
106 placement->num_placement))
107 return true;
108
109 if ((placement->busy_placement != placement->placement ||
110 placement->num_busy_placement > placement->num_placement) &&
111 ttm_resource_places_compat(res, placement->busy_placement,
112 placement->num_busy_placement))
113 return true;
114
115 return false;
116 }
117 EXPORT_SYMBOL(ttm_resource_compat);
118
119 /**
120 * ttm_resource_manager_init
121 *
122 * @man: memory manager object to init
123 * @p_size: size managed area in pages.
124 *
125 * Initialise core parts of a manager object.
126 */
ttm_resource_manager_init(struct ttm_resource_manager * man,unsigned long p_size)127 void ttm_resource_manager_init(struct ttm_resource_manager *man,
128 unsigned long p_size)
129 {
130 unsigned i;
131
132 spin_lock_init(&man->move_lock);
133 man->size = p_size;
134
135 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
136 INIT_LIST_HEAD(&man->lru[i]);
137 man->move = NULL;
138 }
139 EXPORT_SYMBOL(ttm_resource_manager_init);
140
141 /*
142 * ttm_resource_manager_evict_all
143 *
144 * @bdev - device to use
145 * @man - manager to use
146 *
147 * Evict all the objects out of a memory manager until it is empty.
148 * Part of memory manager cleanup sequence.
149 */
ttm_resource_manager_evict_all(struct ttm_device * bdev,struct ttm_resource_manager * man)150 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
151 struct ttm_resource_manager *man)
152 {
153 struct ttm_operation_ctx ctx = {
154 .interruptible = false,
155 .no_wait_gpu = false,
156 .force_alloc = true
157 };
158 struct dma_fence *fence;
159 int ret;
160 unsigned i;
161
162 /*
163 * Can't use standard list traversal since we're unlocking.
164 */
165
166 spin_lock(&bdev->lru_lock);
167 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
168 while (!list_empty(&man->lru[i])) {
169 spin_unlock(&bdev->lru_lock);
170 ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
171 NULL);
172 if (ret)
173 return ret;
174 spin_lock(&bdev->lru_lock);
175 }
176 }
177 spin_unlock(&bdev->lru_lock);
178
179 spin_lock(&man->move_lock);
180 fence = dma_fence_get(man->move);
181 spin_unlock(&man->move_lock);
182
183 if (fence) {
184 ret = dma_fence_wait(fence, false);
185 dma_fence_put(fence);
186 if (ret)
187 return ret;
188 }
189
190 return 0;
191 }
192 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
193
194 /**
195 * ttm_resource_manager_debug
196 *
197 * @man: manager type to dump.
198 * @p: printer to use for debug.
199 */
ttm_resource_manager_debug(struct ttm_resource_manager * man,struct drm_printer * p)200 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
201 struct drm_printer *p)
202 {
203 drm_printf(p, " use_type: %d\n", man->use_type);
204 drm_printf(p, " use_tt: %d\n", man->use_tt);
205 drm_printf(p, " size: %llu\n", man->size);
206 if (man->func->debug)
207 man->func->debug(man, p);
208 }
209 EXPORT_SYMBOL(ttm_resource_manager_debug);
210
ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter * iter,struct dma_buf_map * dmap,pgoff_t i)211 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
212 struct dma_buf_map *dmap,
213 pgoff_t i)
214 {
215 struct ttm_kmap_iter_iomap *iter_io =
216 container_of(iter, typeof(*iter_io), base);
217 void __iomem *addr;
218
219 retry:
220 while (i >= iter_io->cache.end) {
221 iter_io->cache.sg = iter_io->cache.sg ?
222 sg_next(iter_io->cache.sg) : iter_io->st->sgl;
223 iter_io->cache.i = iter_io->cache.end;
224 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
225 PAGE_SHIFT;
226 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
227 iter_io->start;
228 }
229
230 if (i < iter_io->cache.i) {
231 iter_io->cache.end = 0;
232 iter_io->cache.sg = NULL;
233 goto retry;
234 }
235
236 addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
237 (((resource_size_t)i - iter_io->cache.i)
238 << PAGE_SHIFT));
239 dma_buf_map_set_vaddr_iomem(dmap, addr);
240 }
241
ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter * iter,struct dma_buf_map * map)242 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
243 struct dma_buf_map *map)
244 {
245 io_mapping_unmap_local(map->vaddr_iomem);
246 }
247
248 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
249 .map_local = ttm_kmap_iter_iomap_map_local,
250 .unmap_local = ttm_kmap_iter_iomap_unmap_local,
251 .maps_tt = false,
252 };
253
254 /**
255 * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
256 * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
257 * @iomap: The struct io_mapping representing the underlying linear io_memory.
258 * @st: sg_table into @iomap, representing the memory of the struct
259 * ttm_resource.
260 * @start: Offset that needs to be subtracted from @st to make
261 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
262 *
263 * Return: Pointer to the embedded struct ttm_kmap_iter.
264 */
265 struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap * iter_io,struct io_mapping * iomap,struct sg_table * st,resource_size_t start)266 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
267 struct io_mapping *iomap,
268 struct sg_table *st,
269 resource_size_t start)
270 {
271 iter_io->base.ops = &ttm_kmap_iter_io_ops;
272 iter_io->iomap = iomap;
273 iter_io->st = st;
274 iter_io->start = start;
275 memset(&iter_io->cache, 0, sizeof(iter_io->cache));
276
277 return &iter_io->base;
278 }
279 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
280
281 /**
282 * DOC: Linear io iterator
283 *
284 * This code should die in the not too near future. Best would be if we could
285 * make io-mapping use memremap for all io memory, and have memremap
286 * implement a kmap_local functionality. We could then strip a huge amount of
287 * code. These linear io iterators are implemented to mimic old functionality,
288 * and they don't use kmap_local semantics at all internally. Rather ioremap or
289 * friends, and at least on 32-bit they add global TLB flushes and points
290 * of failure.
291 */
292
ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter * iter,struct dma_buf_map * dmap,pgoff_t i)293 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
294 struct dma_buf_map *dmap,
295 pgoff_t i)
296 {
297 struct ttm_kmap_iter_linear_io *iter_io =
298 container_of(iter, typeof(*iter_io), base);
299
300 *dmap = iter_io->dmap;
301 dma_buf_map_incr(dmap, i * PAGE_SIZE);
302 }
303
304 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
305 .map_local = ttm_kmap_iter_linear_io_map_local,
306 .maps_tt = false,
307 };
308
309 /**
310 * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
311 * @iter_io: The iterator to initialize
312 * @bdev: The TTM device
313 * @mem: The ttm resource representing the iomap.
314 *
315 * This function is for internal TTM use only. It sets up a memcpy kmap iterator
316 * pointing at a linear chunk of io memory.
317 *
318 * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
319 * failure.
320 */
321 struct ttm_kmap_iter *
ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)322 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
323 struct ttm_device *bdev,
324 struct ttm_resource *mem)
325 {
326 int ret;
327
328 ret = ttm_mem_io_reserve(bdev, mem);
329 if (ret)
330 goto out_err;
331 if (!mem->bus.is_iomem) {
332 ret = -EINVAL;
333 goto out_io_free;
334 }
335
336 if (mem->bus.addr) {
337 dma_buf_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
338 iter_io->needs_unmap = false;
339 } else {
340 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
341
342 iter_io->needs_unmap = true;
343 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
344 if (mem->bus.caching == ttm_write_combined)
345 dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
346 ioremap_wc(mem->bus.offset,
347 bus_size));
348 else if (mem->bus.caching == ttm_cached)
349 dma_buf_map_set_vaddr(&iter_io->dmap,
350 memremap(mem->bus.offset, bus_size,
351 MEMREMAP_WB |
352 MEMREMAP_WT |
353 MEMREMAP_WC));
354
355 /* If uncached requested or if mapping cached or wc failed */
356 if (dma_buf_map_is_null(&iter_io->dmap))
357 dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
358 ioremap(mem->bus.offset,
359 bus_size));
360
361 if (dma_buf_map_is_null(&iter_io->dmap)) {
362 ret = -ENOMEM;
363 goto out_io_free;
364 }
365 }
366
367 iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
368 return &iter_io->base;
369
370 out_io_free:
371 ttm_mem_io_free(bdev, mem);
372 out_err:
373 return ERR_PTR(ret);
374 }
375
376 /**
377 * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
378 * @iter_io: The iterator to initialize
379 * @bdev: The TTM device
380 * @mem: The ttm resource representing the iomap.
381 *
382 * This function is for internal TTM use only. It cleans up a memcpy kmap
383 * iterator initialized by ttm_kmap_iter_linear_io_init.
384 */
385 void
ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io * iter_io,struct ttm_device * bdev,struct ttm_resource * mem)386 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
387 struct ttm_device *bdev,
388 struct ttm_resource *mem)
389 {
390 if (iter_io->needs_unmap && dma_buf_map_is_set(&iter_io->dmap)) {
391 if (iter_io->dmap.is_iomem)
392 iounmap(iter_io->dmap.vaddr_iomem);
393 else
394 memunmap(iter_io->dmap.vaddr);
395 }
396
397 ttm_mem_io_free(bdev, mem);
398 }
399