Lines Matching refs:bo
48 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) in ttm_bo_default_destroy() argument
50 kfree(bo); in ttm_bo_default_destroy()
53 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, in ttm_bo_mem_space_debug() argument
61 bo, bo->resource->num_pages, bo->base.size >> 10, in ttm_bo_mem_space_debug()
62 bo->base.size >> 20); in ttm_bo_mem_space_debug()
67 man = ttm_manager_type(bo->bdev, mem_type); in ttm_bo_mem_space_debug()
72 static inline void ttm_bo_move_to_pinned(struct ttm_buffer_object *bo) in ttm_bo_move_to_pinned() argument
74 struct ttm_device *bdev = bo->bdev; in ttm_bo_move_to_pinned()
76 list_move_tail(&bo->lru, &bdev->pinned); in ttm_bo_move_to_pinned()
79 bdev->funcs->del_from_lru_notify(bo); in ttm_bo_move_to_pinned()
82 static inline void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) in ttm_bo_del_from_lru() argument
84 struct ttm_device *bdev = bo->bdev; in ttm_bo_del_from_lru()
86 list_del_init(&bo->lru); in ttm_bo_del_from_lru()
89 bdev->funcs->del_from_lru_notify(bo); in ttm_bo_del_from_lru()
93 struct ttm_buffer_object *bo) in ttm_bo_bulk_move_set_pos() argument
96 pos->first = bo; in ttm_bo_bulk_move_set_pos()
97 pos->last = bo; in ttm_bo_bulk_move_set_pos()
100 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, in ttm_bo_move_to_lru_tail() argument
104 struct ttm_device *bdev = bo->bdev; in ttm_bo_move_to_lru_tail()
107 if (!bo->deleted) in ttm_bo_move_to_lru_tail()
108 dma_resv_assert_held(bo->base.resv); in ttm_bo_move_to_lru_tail()
110 if (bo->pin_count) { in ttm_bo_move_to_lru_tail()
111 ttm_bo_move_to_pinned(bo); in ttm_bo_move_to_lru_tail()
119 list_move_tail(&bo->lru, &man->lru[bo->priority]); in ttm_bo_move_to_lru_tail()
122 bdev->funcs->del_from_lru_notify(bo); in ttm_bo_move_to_lru_tail()
124 if (bulk && !bo->pin_count) { in ttm_bo_move_to_lru_tail()
125 switch (bo->resource->mem_type) { in ttm_bo_move_to_lru_tail()
127 ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); in ttm_bo_move_to_lru_tail()
131 ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo); in ttm_bo_move_to_lru_tail()
174 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, in ttm_bo_handle_move_mem() argument
180 struct ttm_device *bdev = bo->bdev; in ttm_bo_handle_move_mem()
183 old_man = ttm_manager_type(bdev, bo->resource->mem_type); in ttm_bo_handle_move_mem()
186 ttm_bo_unmap_virtual(bo); in ttm_bo_handle_move_mem()
196 ret = ttm_tt_create(bo, old_man->use_tt); in ttm_bo_handle_move_mem()
201 ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx); in ttm_bo_handle_move_mem()
207 ret = bdev->funcs->move(bo, evict, ctx, mem, hop); in ttm_bo_handle_move_mem()
214 ctx->bytes_moved += bo->base.size; in ttm_bo_handle_move_mem()
218 new_man = ttm_manager_type(bdev, bo->resource->mem_type); in ttm_bo_handle_move_mem()
220 ttm_bo_tt_destroy(bo); in ttm_bo_handle_move_mem()
233 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) in ttm_bo_cleanup_memtype_use() argument
235 if (bo->bdev->funcs->delete_mem_notify) in ttm_bo_cleanup_memtype_use()
236 bo->bdev->funcs->delete_mem_notify(bo); in ttm_bo_cleanup_memtype_use()
238 ttm_bo_tt_destroy(bo); in ttm_bo_cleanup_memtype_use()
239 ttm_resource_free(bo, &bo->resource); in ttm_bo_cleanup_memtype_use()
242 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) in ttm_bo_individualize_resv() argument
246 if (bo->base.resv == &bo->base._resv) in ttm_bo_individualize_resv()
249 BUG_ON(!dma_resv_trylock(&bo->base._resv)); in ttm_bo_individualize_resv()
251 r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); in ttm_bo_individualize_resv()
252 dma_resv_unlock(&bo->base._resv); in ttm_bo_individualize_resv()
256 if (bo->type != ttm_bo_type_sg) { in ttm_bo_individualize_resv()
261 spin_lock(&bo->bdev->lru_lock); in ttm_bo_individualize_resv()
262 bo->base.resv = &bo->base._resv; in ttm_bo_individualize_resv()
263 spin_unlock(&bo->bdev->lru_lock); in ttm_bo_individualize_resv()
269 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) in ttm_bo_flush_all_fences() argument
271 struct dma_resv *resv = &bo->base._resv; in ttm_bo_flush_all_fences()
297 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, in ttm_bo_cleanup_refs() argument
301 struct dma_resv *resv = &bo->base._resv; in ttm_bo_cleanup_refs()
313 dma_resv_unlock(bo->base.resv); in ttm_bo_cleanup_refs()
314 spin_unlock(&bo->bdev->lru_lock); in ttm_bo_cleanup_refs()
324 spin_lock(&bo->bdev->lru_lock); in ttm_bo_cleanup_refs()
325 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { in ttm_bo_cleanup_refs()
334 spin_unlock(&bo->bdev->lru_lock); in ttm_bo_cleanup_refs()
340 if (ret || unlikely(list_empty(&bo->ddestroy))) { in ttm_bo_cleanup_refs()
342 dma_resv_unlock(bo->base.resv); in ttm_bo_cleanup_refs()
343 spin_unlock(&bo->bdev->lru_lock); in ttm_bo_cleanup_refs()
347 ttm_bo_move_to_pinned(bo); in ttm_bo_cleanup_refs()
348 list_del_init(&bo->ddestroy); in ttm_bo_cleanup_refs()
349 spin_unlock(&bo->bdev->lru_lock); in ttm_bo_cleanup_refs()
350 ttm_bo_cleanup_memtype_use(bo); in ttm_bo_cleanup_refs()
353 dma_resv_unlock(bo->base.resv); in ttm_bo_cleanup_refs()
355 ttm_bo_put(bo); in ttm_bo_cleanup_refs()
373 struct ttm_buffer_object *bo; in ttm_bo_delayed_delete() local
375 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, in ttm_bo_delayed_delete()
377 list_move_tail(&bo->ddestroy, &removed); in ttm_bo_delayed_delete()
378 if (!ttm_bo_get_unless_zero(bo)) in ttm_bo_delayed_delete()
381 if (remove_all || bo->base.resv != &bo->base._resv) { in ttm_bo_delayed_delete()
383 dma_resv_lock(bo->base.resv, NULL); in ttm_bo_delayed_delete()
386 ttm_bo_cleanup_refs(bo, false, !remove_all, true); in ttm_bo_delayed_delete()
388 } else if (dma_resv_trylock(bo->base.resv)) { in ttm_bo_delayed_delete()
389 ttm_bo_cleanup_refs(bo, false, !remove_all, true); in ttm_bo_delayed_delete()
394 ttm_bo_put(bo); in ttm_bo_delayed_delete()
406 struct ttm_buffer_object *bo = in ttm_bo_release() local
408 struct ttm_device *bdev = bo->bdev; in ttm_bo_release()
411 WARN_ON_ONCE(bo->pin_count); in ttm_bo_release()
413 if (!bo->deleted) { in ttm_bo_release()
414 ret = ttm_bo_individualize_resv(bo); in ttm_bo_release()
419 dma_resv_wait_timeout(bo->base.resv, true, false, in ttm_bo_release()
423 if (bo->bdev->funcs->release_notify) in ttm_bo_release()
424 bo->bdev->funcs->release_notify(bo); in ttm_bo_release()
426 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); in ttm_bo_release()
427 ttm_mem_io_free(bdev, bo->resource); in ttm_bo_release()
430 if (!dma_resv_test_signaled(bo->base.resv, true) || in ttm_bo_release()
431 !dma_resv_trylock(bo->base.resv)) { in ttm_bo_release()
433 ttm_bo_flush_all_fences(bo); in ttm_bo_release()
434 bo->deleted = true; in ttm_bo_release()
436 spin_lock(&bo->bdev->lru_lock); in ttm_bo_release()
446 if (bo->pin_count) { in ttm_bo_release()
447 bo->pin_count = 0; in ttm_bo_release()
448 ttm_bo_move_to_lru_tail(bo, bo->resource, NULL); in ttm_bo_release()
451 kref_init(&bo->kref); in ttm_bo_release()
452 list_add_tail(&bo->ddestroy, &bdev->ddestroy); in ttm_bo_release()
453 spin_unlock(&bo->bdev->lru_lock); in ttm_bo_release()
460 spin_lock(&bo->bdev->lru_lock); in ttm_bo_release()
461 ttm_bo_del_from_lru(bo); in ttm_bo_release()
462 list_del(&bo->ddestroy); in ttm_bo_release()
463 spin_unlock(&bo->bdev->lru_lock); in ttm_bo_release()
465 ttm_bo_cleanup_memtype_use(bo); in ttm_bo_release()
466 dma_resv_unlock(bo->base.resv); in ttm_bo_release()
469 dma_fence_put(bo->moving); in ttm_bo_release()
470 bo->destroy(bo); in ttm_bo_release()
473 void ttm_bo_put(struct ttm_buffer_object *bo) in ttm_bo_put() argument
475 kref_put(&bo->kref, ttm_bo_release); in ttm_bo_put()
493 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, in ttm_bo_bounce_temp_buffer() argument
506 ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); in ttm_bo_bounce_temp_buffer()
510 ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL); in ttm_bo_bounce_temp_buffer()
512 ttm_resource_free(bo, &hop_mem); in ttm_bo_bounce_temp_buffer()
518 static int ttm_bo_evict(struct ttm_buffer_object *bo, in ttm_bo_evict() argument
521 struct ttm_device *bdev = bo->bdev; in ttm_bo_evict()
529 dma_resv_assert_held(bo->base.resv); in ttm_bo_evict()
533 bdev->funcs->evict_flags(bo, &placement); in ttm_bo_evict()
536 ret = ttm_bo_wait(bo, true, false); in ttm_bo_evict()
544 return ttm_bo_pipeline_gutting(bo); in ttm_bo_evict()
547 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); in ttm_bo_evict()
551 bo); in ttm_bo_evict()
552 ttm_bo_mem_space_debug(bo, &placement); in ttm_bo_evict()
558 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); in ttm_bo_evict()
560 ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop); in ttm_bo_evict()
563 ttm_resource_free(bo, &evict_mem); in ttm_bo_evict()
573 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, in ttm_bo_eviction_valuable() argument
576 dma_resv_assert_held(bo->base.resv); in ttm_bo_eviction_valuable()
577 if (bo->resource->mem_type == TTM_PL_SYSTEM) in ttm_bo_eviction_valuable()
583 if (place->fpfn >= (bo->resource->start + bo->resource->num_pages) || in ttm_bo_eviction_valuable()
584 (place->lpfn && place->lpfn <= bo->resource->start)) in ttm_bo_eviction_valuable()
601 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, in ttm_bo_evict_swapout_allowable() argument
608 if (bo->base.resv == ctx->resv) { in ttm_bo_evict_swapout_allowable()
609 dma_resv_assert_held(bo->base.resv); in ttm_bo_evict_swapout_allowable()
616 ret = dma_resv_trylock(bo->base.resv); in ttm_bo_evict_swapout_allowable()
622 if (ret && place && (bo->resource->mem_type != place->mem_type || in ttm_bo_evict_swapout_allowable()
623 !bo->bdev->funcs->eviction_valuable(bo, place))) { in ttm_bo_evict_swapout_allowable()
626 dma_resv_unlock(bo->base.resv); in ttm_bo_evict_swapout_allowable()
675 struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; in ttm_mem_evict_first() local
682 list_for_each_entry(bo, &man->lru[i], lru) { in ttm_mem_evict_first()
685 if (!ttm_bo_evict_swapout_allowable(bo, ctx, place, in ttm_mem_evict_first()
688 dma_resv_locking_ctx(bo->base.resv)) in ttm_mem_evict_first()
689 busy_bo = bo; in ttm_mem_evict_first()
693 if (!ttm_bo_get_unless_zero(bo)) { in ttm_mem_evict_first()
695 dma_resv_unlock(bo->base.resv); in ttm_mem_evict_first()
702 if (&bo->lru != &man->lru[i]) in ttm_mem_evict_first()
705 bo = NULL; in ttm_mem_evict_first()
708 if (!bo) { in ttm_mem_evict_first()
718 if (bo->deleted) { in ttm_mem_evict_first()
719 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, in ttm_mem_evict_first()
721 ttm_bo_put(bo); in ttm_mem_evict_first()
727 ret = ttm_bo_evict(bo, ctx); in ttm_mem_evict_first()
729 ttm_bo_unreserve(bo); in ttm_mem_evict_first()
731 ttm_bo_put(bo); in ttm_mem_evict_first()
740 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, in ttm_bo_add_move_fence() argument
761 dma_resv_add_shared_fence(bo->base.resv, fence); in ttm_bo_add_move_fence()
763 ret = dma_resv_reserve_shared(bo->base.resv, 1); in ttm_bo_add_move_fence()
769 dma_fence_put(bo->moving); in ttm_bo_add_move_fence()
770 bo->moving = fence; in ttm_bo_add_move_fence()
778 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, in ttm_bo_mem_force_space() argument
783 struct ttm_device *bdev = bo->bdev; in ttm_bo_mem_force_space()
789 ticket = dma_resv_locking_ctx(bo->base.resv); in ttm_bo_mem_force_space()
791 ret = ttm_resource_alloc(bo, place, mem); in ttm_bo_mem_force_space()
802 return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); in ttm_bo_mem_force_space()
813 int ttm_bo_mem_space(struct ttm_buffer_object *bo, in ttm_bo_mem_space() argument
818 struct ttm_device *bdev = bo->bdev; in ttm_bo_mem_space()
822 ret = dma_resv_reserve_shared(bo->base.resv, 1); in ttm_bo_mem_space()
835 ret = ttm_resource_alloc(bo, place, mem); in ttm_bo_mem_space()
841 ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); in ttm_bo_mem_space()
843 ttm_resource_free(bo, mem); in ttm_bo_mem_space()
861 ret = ttm_bo_mem_force_space(bo, place, mem, ctx); in ttm_bo_mem_space()
876 if (bo->resource->mem_type == TTM_PL_SYSTEM && !bo->pin_count) in ttm_bo_mem_space()
877 ttm_bo_move_to_lru_tail_unlocked(bo); in ttm_bo_mem_space()
883 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, in ttm_bo_move_buffer() argument
891 dma_resv_assert_held(bo->base.resv); in ttm_bo_move_buffer()
902 ret = ttm_bo_mem_space(bo, placement, &mem, ctx); in ttm_bo_move_buffer()
906 ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop); in ttm_bo_move_buffer()
908 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop); in ttm_bo_move_buffer()
916 ttm_resource_free(bo, &mem); in ttm_bo_move_buffer()
920 int ttm_bo_validate(struct ttm_buffer_object *bo, in ttm_bo_validate() argument
926 dma_resv_assert_held(bo->base.resv); in ttm_bo_validate()
932 return ttm_bo_pipeline_gutting(bo); in ttm_bo_validate()
937 if (!ttm_resource_compat(bo->resource, placement)) { in ttm_bo_validate()
938 ret = ttm_bo_move_buffer(bo, placement, ctx); in ttm_bo_validate()
945 if (bo->resource->mem_type == TTM_PL_SYSTEM) { in ttm_bo_validate()
946 ret = ttm_tt_create(bo, true); in ttm_bo_validate()
955 struct ttm_buffer_object *bo, in ttm_bo_init_reserved() argument
969 bo->destroy = destroy ? destroy : ttm_bo_default_destroy; in ttm_bo_init_reserved()
971 kref_init(&bo->kref); in ttm_bo_init_reserved()
972 INIT_LIST_HEAD(&bo->lru); in ttm_bo_init_reserved()
973 INIT_LIST_HEAD(&bo->ddestroy); in ttm_bo_init_reserved()
974 bo->bdev = bdev; in ttm_bo_init_reserved()
975 bo->type = type; in ttm_bo_init_reserved()
976 bo->page_alignment = page_alignment; in ttm_bo_init_reserved()
977 bo->moving = NULL; in ttm_bo_init_reserved()
978 bo->pin_count = 0; in ttm_bo_init_reserved()
979 bo->sg = sg; in ttm_bo_init_reserved()
981 bo->base.resv = resv; in ttm_bo_init_reserved()
982 dma_resv_assert_held(bo->base.resv); in ttm_bo_init_reserved()
984 bo->base.resv = &bo->base._resv; in ttm_bo_init_reserved()
988 ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource); in ttm_bo_init_reserved()
990 ttm_bo_put(bo); in ttm_bo_init_reserved()
998 if (bo->type == ttm_bo_type_device || in ttm_bo_init_reserved()
999 bo->type == ttm_bo_type_sg) in ttm_bo_init_reserved()
1000 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, in ttm_bo_init_reserved()
1001 bo->resource->num_pages); in ttm_bo_init_reserved()
1007 locked = dma_resv_trylock(bo->base.resv); in ttm_bo_init_reserved()
1012 ret = ttm_bo_validate(bo, placement, ctx); in ttm_bo_init_reserved()
1016 ttm_bo_unreserve(bo); in ttm_bo_init_reserved()
1018 ttm_bo_put(bo); in ttm_bo_init_reserved()
1022 ttm_bo_move_to_lru_tail_unlocked(bo); in ttm_bo_init_reserved()
1029 struct ttm_buffer_object *bo, in ttm_bo_init() argument
1042 ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, in ttm_bo_init()
1048 ttm_bo_unreserve(bo); in ttm_bo_init()
1058 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) in ttm_bo_unmap_virtual() argument
1060 struct ttm_device *bdev = bo->bdev; in ttm_bo_unmap_virtual()
1062 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); in ttm_bo_unmap_virtual()
1063 ttm_mem_io_free(bdev, bo->resource); in ttm_bo_unmap_virtual()
1067 int ttm_bo_wait(struct ttm_buffer_object *bo, in ttm_bo_wait() argument
1073 if (dma_resv_test_signaled(bo->base.resv, true)) in ttm_bo_wait()
1079 timeout = dma_resv_wait_timeout(bo->base.resv, true, interruptible, in ttm_bo_wait()
1087 dma_resv_add_excl_fence(bo->base.resv, NULL); in ttm_bo_wait()
1092 int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, in ttm_bo_swapout() argument
1106 place.mem_type = bo->resource->mem_type; in ttm_bo_swapout()
1107 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL)) in ttm_bo_swapout()
1110 if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) || in ttm_bo_swapout()
1111 bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL || in ttm_bo_swapout()
1112 bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED || in ttm_bo_swapout()
1113 !ttm_bo_get_unless_zero(bo)) { in ttm_bo_swapout()
1115 dma_resv_unlock(bo->base.resv); in ttm_bo_swapout()
1119 if (bo->deleted) { in ttm_bo_swapout()
1120 ret = ttm_bo_cleanup_refs(bo, false, false, locked); in ttm_bo_swapout()
1121 ttm_bo_put(bo); in ttm_bo_swapout()
1125 ttm_bo_move_to_pinned(bo); in ttm_bo_swapout()
1127 spin_unlock(&bo->bdev->lru_lock); in ttm_bo_swapout()
1132 if (bo->resource->mem_type != TTM_PL_SYSTEM) { in ttm_bo_swapout()
1139 ret = ttm_resource_alloc(bo, &place, &evict_mem); in ttm_bo_swapout()
1143 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, &ctx, &hop); in ttm_bo_swapout()
1153 ret = ttm_bo_wait(bo, false, false); in ttm_bo_swapout()
1157 ttm_bo_unmap_virtual(bo); in ttm_bo_swapout()
1163 if (bo->bdev->funcs->swap_notify) in ttm_bo_swapout()
1164 bo->bdev->funcs->swap_notify(bo); in ttm_bo_swapout()
1166 if (ttm_tt_is_populated(bo->ttm)) in ttm_bo_swapout()
1167 ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags); in ttm_bo_swapout()
1175 dma_resv_unlock(bo->base.resv); in ttm_bo_swapout()
1176 ttm_bo_put(bo); in ttm_bo_swapout()
1180 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) in ttm_bo_tt_destroy() argument
1182 if (bo->ttm == NULL) in ttm_bo_tt_destroy()
1185 ttm_tt_unpopulate(bo->bdev, bo->ttm); in ttm_bo_tt_destroy()
1186 ttm_tt_destroy(bo->bdev, bo->ttm); in ttm_bo_tt_destroy()
1187 bo->ttm = NULL; in ttm_bo_tt_destroy()