Lines Matching refs:pdd
75 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
88 struct kfd_process_device *pdd; member
102 struct kfd_process_device *pdd; in kfd_sdma_activity_worker() local
115 pdd = workarea->pdd; in kfd_sdma_activity_worker()
116 if (!pdd) in kfd_sdma_activity_worker()
118 dqm = pdd->dev->dqm; in kfd_sdma_activity_worker()
119 qpd = &pdd->qpd; in kfd_sdma_activity_worker()
174 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
184 mm = get_task_mm(pdd->process->lead_thread); in kfd_sdma_activity_worker()
211 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; in kfd_sdma_activity_worker()
273 struct kfd_process_device *pdd = NULL; in kfd_get_cu_occupancy() local
275 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy); in kfd_get_cu_occupancy()
276 dev = pdd->dev; in kfd_get_cu_occupancy()
281 proc = pdd->process; in kfd_get_cu_occupancy()
282 if (pdd->qpd.queue_count == 0) { in kfd_get_cu_occupancy()
308 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, in kfd_procfs_show() local
310 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage)); in kfd_procfs_show()
312 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, in kfd_procfs_show() local
319 sdma_activity_work_handler.pdd = pdd; in kfd_procfs_show()
399 struct kfd_process_device *pdd = container_of(attr, in kfd_procfs_stats_show() local
404 evict_jiffies = atomic64_read(&pdd->evict_duration_counter); in kfd_procfs_stats_show()
424 struct kfd_process_device *pdd; in kfd_sysfs_counters_show() local
427 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
429 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults)); in kfd_sysfs_counters_show()
432 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
434 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in)); in kfd_sysfs_counters_show()
437 pdd = container_of(attr, struct kfd_process_device, in kfd_sysfs_counters_show()
439 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out)); in kfd_sysfs_counters_show()
550 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_stats() local
553 "stats_%u", pdd->dev->id); in kfd_procfs_add_sysfs_stats()
554 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
555 if (!pdd->kobj_stats) in kfd_procfs_add_sysfs_stats()
558 ret = kobject_init_and_add(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
566 kobject_put(pdd->kobj_stats); in kfd_procfs_add_sysfs_stats()
567 pdd->kobj_stats = NULL; in kfd_procfs_add_sysfs_stats()
571 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict, in kfd_procfs_add_sysfs_stats()
574 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_procfs_add_sysfs_stats()
575 kfd_sysfs_create_file(pdd->kobj_stats, in kfd_procfs_add_sysfs_stats()
576 &pdd->attr_cu_occupancy, in kfd_procfs_add_sysfs_stats()
598 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_counters() local
602 "counters_%u", pdd->dev->id); in kfd_procfs_add_sysfs_counters()
616 pdd->kobj_counters = kobj_counters; in kfd_procfs_add_sysfs_counters()
617 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults, in kfd_procfs_add_sysfs_counters()
619 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in, in kfd_procfs_add_sysfs_counters()
621 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out, in kfd_procfs_add_sysfs_counters()
639 struct kfd_process_device *pdd = p->pdds[i]; in kfd_procfs_add_sysfs_files() local
641 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u", in kfd_procfs_add_sysfs_files()
642 pdd->dev->id); in kfd_procfs_add_sysfs_files()
643 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram, in kfd_procfs_add_sysfs_files()
644 pdd->vram_filename); in kfd_procfs_add_sysfs_files()
646 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u", in kfd_procfs_add_sysfs_files()
647 pdd->dev->id); in kfd_procfs_add_sysfs_files()
648 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma, in kfd_procfs_add_sysfs_files()
649 pdd->sdma_filename); in kfd_procfs_add_sysfs_files()
690 struct kfd_process_device *pdd, void *kptr) in kfd_process_free_gpuvm() argument
692 struct kfd_dev *dev = pdd->dev; in kfd_process_free_gpuvm()
699 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv); in kfd_process_free_gpuvm()
700 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv, in kfd_process_free_gpuvm()
710 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, in kfd_process_alloc_gpuvm() argument
714 struct kfd_dev *kdev = pdd->dev; in kfd_process_alloc_gpuvm()
718 pdd->drm_priv, mem, NULL, flags); in kfd_process_alloc_gpuvm()
723 pdd->drm_priv, NULL); in kfd_process_alloc_gpuvm()
745 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->kgd, *mem, pdd->drm_priv); in kfd_process_alloc_gpuvm()
748 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, *mem, pdd->drm_priv, in kfd_process_alloc_gpuvm()
762 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd) in kfd_process_device_reserve_ib_mem() argument
764 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_reserve_ib_mem()
777 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags, in kfd_process_device_reserve_ib_mem()
788 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd) in kfd_process_device_destroy_ib_mem() argument
790 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_ib_mem()
795 kfd_process_free_gpuvm(qpd->ib_mem, pdd, qpd->ib_kaddr); in kfd_process_device_destroy_ib_mem()
924 static void kfd_process_device_free_bos(struct kfd_process_device *pdd) in kfd_process_device_free_bos() argument
926 struct kfd_process *p = pdd->process; in kfd_process_device_free_bos()
935 idr_for_each_entry(&pdd->alloc_idr, mem, id) { in kfd_process_device_free_bos()
946 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, in kfd_process_device_free_bos()
947 pdd->drm_priv, NULL); in kfd_process_device_free_bos()
948 kfd_process_device_remove_obj_handle(pdd, id); in kfd_process_device_free_bos()
958 struct kfd_process_device *pdd; in kfd_process_kunmap_signal_bo() local
968 pdd = kfd_get_process_device_data(kdev, p); in kfd_process_kunmap_signal_bo()
969 if (!pdd) in kfd_process_kunmap_signal_bo()
973 pdd, GET_IDR_HANDLE(p->signal_handle)); in kfd_process_kunmap_signal_bo()
996 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_destroy_pdds() local
999 pdd->dev->id, p->pasid); in kfd_process_destroy_pdds()
1001 kfd_process_device_destroy_cwsr_dgpu(pdd); in kfd_process_destroy_pdds()
1002 kfd_process_device_destroy_ib_mem(pdd); in kfd_process_destroy_pdds()
1004 if (pdd->drm_file) { in kfd_process_destroy_pdds()
1006 pdd->dev->kgd, pdd->drm_priv); in kfd_process_destroy_pdds()
1007 fput(pdd->drm_file); in kfd_process_destroy_pdds()
1010 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) in kfd_process_destroy_pdds()
1011 free_pages((unsigned long)pdd->qpd.cwsr_kaddr, in kfd_process_destroy_pdds()
1014 kfree(pdd->qpd.doorbell_bitmap); in kfd_process_destroy_pdds()
1015 idr_destroy(&pdd->alloc_idr); in kfd_process_destroy_pdds()
1017 kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index); in kfd_process_destroy_pdds()
1023 if (pdd->runtime_inuse) { in kfd_process_destroy_pdds()
1024 pm_runtime_mark_last_busy(pdd->dev->ddev->dev); in kfd_process_destroy_pdds()
1025 pm_runtime_put_autosuspend(pdd->dev->ddev->dev); in kfd_process_destroy_pdds()
1026 pdd->runtime_inuse = false; in kfd_process_destroy_pdds()
1029 kfree(pdd); in kfd_process_destroy_pdds()
1037 struct kfd_process_device *pdd; in kfd_process_remove_sysfs() local
1049 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1051 sysfs_remove_file(p->kobj, &pdd->attr_vram); in kfd_process_remove_sysfs()
1052 sysfs_remove_file(p->kobj, &pdd->attr_sdma); in kfd_process_remove_sysfs()
1054 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict); in kfd_process_remove_sysfs()
1055 if (pdd->dev->kfd2kgd->get_cu_occupancy) in kfd_process_remove_sysfs()
1056 sysfs_remove_file(pdd->kobj_stats, in kfd_process_remove_sysfs()
1057 &pdd->attr_cu_occupancy); in kfd_process_remove_sysfs()
1058 kobject_del(pdd->kobj_stats); in kfd_process_remove_sysfs()
1059 kobject_put(pdd->kobj_stats); in kfd_process_remove_sysfs()
1060 pdd->kobj_stats = NULL; in kfd_process_remove_sysfs()
1064 pdd = p->pdds[i]; in kfd_process_remove_sysfs()
1066 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults); in kfd_process_remove_sysfs()
1067 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in); in kfd_process_remove_sysfs()
1068 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out); in kfd_process_remove_sysfs()
1069 kobject_del(pdd->kobj_counters); in kfd_process_remove_sysfs()
1070 kobject_put(pdd->kobj_counters); in kfd_process_remove_sysfs()
1071 pdd->kobj_counters = NULL; in kfd_process_remove_sysfs()
1232 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) in kfd_process_device_init_cwsr_dgpu() argument
1234 struct kfd_dev *dev = pdd->dev; in kfd_process_device_init_cwsr_dgpu()
1235 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_init_cwsr_dgpu()
1247 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, in kfd_process_device_init_cwsr_dgpu()
1265 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd) in kfd_process_device_destroy_cwsr_dgpu() argument
1267 struct kfd_dev *dev = pdd->dev; in kfd_process_device_destroy_cwsr_dgpu()
1268 struct qcm_process_device *qpd = &pdd->qpd; in kfd_process_device_destroy_cwsr_dgpu()
1273 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, qpd->cwsr_kaddr); in kfd_process_device_destroy_cwsr_dgpu()
1475 struct kfd_process_device *pdd = NULL; in kfd_create_process_device_data() local
1479 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); in kfd_create_process_device_data()
1480 if (!pdd) in kfd_create_process_device_data()
1483 if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) { in kfd_create_process_device_data()
1488 if (init_doorbell_bitmap(&pdd->qpd, dev)) { in kfd_create_process_device_data()
1493 pdd->dev = dev; in kfd_create_process_device_data()
1494 INIT_LIST_HEAD(&pdd->qpd.queues_list); in kfd_create_process_device_data()
1495 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); in kfd_create_process_device_data()
1496 pdd->qpd.dqm = dev->dqm; in kfd_create_process_device_data()
1497 pdd->qpd.pqm = &p->pqm; in kfd_create_process_device_data()
1498 pdd->qpd.evicted = 0; in kfd_create_process_device_data()
1499 pdd->qpd.mapped_gws_queue = false; in kfd_create_process_device_data()
1500 pdd->process = p; in kfd_create_process_device_data()
1501 pdd->bound = PDD_UNBOUND; in kfd_create_process_device_data()
1502 pdd->already_dequeued = false; in kfd_create_process_device_data()
1503 pdd->runtime_inuse = false; in kfd_create_process_device_data()
1504 pdd->vram_usage = 0; in kfd_create_process_device_data()
1505 pdd->sdma_past_activity_counter = 0; in kfd_create_process_device_data()
1506 atomic64_set(&pdd->evict_duration_counter, 0); in kfd_create_process_device_data()
1507 p->pdds[p->n_pdds++] = pdd; in kfd_create_process_device_data()
1510 idr_init(&pdd->alloc_idr); in kfd_create_process_device_data()
1512 return pdd; in kfd_create_process_device_data()
1515 kfree(pdd); in kfd_create_process_device_data()
1533 int kfd_process_device_init_vm(struct kfd_process_device *pdd, in kfd_process_device_init_vm() argument
1543 if (pdd->drm_priv) in kfd_process_device_init_vm()
1546 p = pdd->process; in kfd_process_device_init_vm()
1547 dev = pdd->dev; in kfd_process_device_init_vm()
1556 pdd->drm_priv = drm_file->private_data; in kfd_process_device_init_vm()
1558 ret = kfd_process_device_reserve_ib_mem(pdd); in kfd_process_device_init_vm()
1561 ret = kfd_process_device_init_cwsr_dgpu(pdd); in kfd_process_device_init_vm()
1565 pdd->drm_file = drm_file; in kfd_process_device_init_vm()
1571 kfd_process_device_free_bos(pdd); in kfd_process_device_init_vm()
1572 pdd->drm_priv = NULL; in kfd_process_device_init_vm()
1587 struct kfd_process_device *pdd; in kfd_bind_process_to_device() local
1590 pdd = kfd_get_process_device_data(dev, p); in kfd_bind_process_to_device()
1591 if (!pdd) { in kfd_bind_process_to_device()
1596 if (!pdd->drm_priv) in kfd_bind_process_to_device()
1604 if (!pdd->runtime_inuse) { in kfd_bind_process_to_device()
1612 err = kfd_iommu_bind_process_to_device(pdd); in kfd_bind_process_to_device()
1620 pdd->runtime_inuse = true; in kfd_bind_process_to_device()
1622 return pdd; in kfd_bind_process_to_device()
1626 if (!pdd->runtime_inuse) { in kfd_bind_process_to_device()
1637 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, in kfd_process_device_create_obj_handle() argument
1640 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL); in kfd_process_device_create_obj_handle()
1646 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd, in kfd_process_device_translate_handle() argument
1652 return idr_find(&pdd->alloc_idr, handle); in kfd_process_device_translate_handle()
1658 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, in kfd_process_device_remove_obj_handle() argument
1662 idr_remove(&pdd->alloc_idr, handle); in kfd_process_device_remove_obj_handle()
1714 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues() local
1716 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1717 &pdd->qpd); in kfd_process_evict_queues()
1736 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_evict_queues() local
1740 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_evict_queues()
1741 &pdd->qpd)) in kfd_process_evict_queues()
1757 struct kfd_process_device *pdd = p->pdds[i]; in kfd_process_restore_queues() local
1759 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, in kfd_process_restore_queues()
1760 &pdd->qpd); in kfd_process_restore_queues()
1917 struct kfd_process_device *pdd; in kfd_reserved_mem_mmap() local
1925 pdd = kfd_get_process_device_data(dev, process); in kfd_reserved_mem_mmap()
1926 if (!pdd) in kfd_reserved_mem_mmap()
1928 qpd = &pdd->qpd; in kfd_reserved_mem_mmap()
1945 void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type) in kfd_flush_tlb() argument
1947 struct kfd_dev *dev = pdd->dev; in kfd_flush_tlb()
1953 if (pdd->qpd.vmid) in kfd_flush_tlb()
1955 pdd->qpd.vmid); in kfd_flush_tlb()
1958 pdd->process->pasid, type); in kfd_flush_tlb()