Lines Matching refs:kq
40 static bool kq_initialize(struct kernel_queue *kq, struct kfd_dev *dev, in kq_initialize() argument
60 kq->dev = dev; in kq_initialize()
61 kq->nop_packet = nop.u32all; in kq_initialize()
64 kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_DIQ]; in kq_initialize()
67 kq->mqd_mgr = dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]; in kq_initialize()
74 if (!kq->mqd_mgr) in kq_initialize()
84 retval = kfd_gtt_sa_allocate(dev, queue_size, &kq->pq); in kq_initialize()
90 kq->pq_kernel_addr = kq->pq->cpu_ptr; in kq_initialize()
91 kq->pq_gpu_addr = kq->pq->gpu_addr; in kq_initialize()
95 retval = kfd_gtt_sa_allocate(dev, PAGE_SIZE, &kq->eop_mem); in kq_initialize()
99 kq->eop_gpu_addr = kq->eop_mem->gpu_addr; in kq_initialize()
100 kq->eop_kernel_addr = kq->eop_mem->cpu_ptr; in kq_initialize()
102 memset(kq->eop_kernel_addr, 0, PAGE_SIZE); in kq_initialize()
105 retval = kfd_gtt_sa_allocate(dev, sizeof(*kq->rptr_kernel), in kq_initialize()
106 &kq->rptr_mem); in kq_initialize()
111 kq->rptr_kernel = kq->rptr_mem->cpu_ptr; in kq_initialize()
112 kq->rptr_gpu_addr = kq->rptr_mem->gpu_addr; in kq_initialize()
115 &kq->wptr_mem); in kq_initialize()
120 kq->wptr_kernel = kq->wptr_mem->cpu_ptr; in kq_initialize()
121 kq->wptr_gpu_addr = kq->wptr_mem->gpu_addr; in kq_initialize()
123 memset(kq->pq_kernel_addr, 0, queue_size); in kq_initialize()
124 memset(kq->rptr_kernel, 0, sizeof(*kq->rptr_kernel)); in kq_initialize()
125 memset(kq->wptr_kernel, 0, sizeof(*kq->wptr_kernel)); in kq_initialize()
134 prop.queue_address = kq->pq_gpu_addr; in kq_initialize()
135 prop.read_ptr = (uint32_t *) kq->rptr_gpu_addr; in kq_initialize()
136 prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr; in kq_initialize()
137 prop.eop_ring_buffer_address = kq->eop_gpu_addr; in kq_initialize()
140 if (init_queue(&kq->queue, &prop) != 0) in kq_initialize()
143 kq->queue->device = dev; in kq_initialize()
144 kq->queue->process = kfd_get_process(current); in kq_initialize()
146 kq->queue->mqd_mem_obj = kq->mqd_mgr->allocate_mqd(kq->mqd_mgr->dev, in kq_initialize()
147 &kq->queue->properties); in kq_initialize()
148 if (!kq->queue->mqd_mem_obj) in kq_initialize()
150 kq->mqd_mgr->init_mqd(kq->mqd_mgr, &kq->queue->mqd, in kq_initialize()
151 kq->queue->mqd_mem_obj, in kq_initialize()
152 &kq->queue->gart_mqd_addr, in kq_initialize()
153 &kq->queue->properties); in kq_initialize()
157 kq->queue->pipe = KFD_CIK_HIQ_PIPE; in kq_initialize()
158 kq->queue->queue = KFD_CIK_HIQ_QUEUE; in kq_initialize()
159 kq->mqd_mgr->load_mqd(kq->mqd_mgr, kq->queue->mqd, in kq_initialize()
160 kq->queue->pipe, kq->queue->queue, in kq_initialize()
161 &kq->queue->properties, NULL); in kq_initialize()
166 &kq->fence_mem_obj); in kq_initialize()
171 kq->fence_kernel_address = kq->fence_mem_obj->cpu_ptr; in kq_initialize()
172 kq->fence_gpu_addr = kq->fence_mem_obj->gpu_addr; in kq_initialize()
175 print_queue(kq->queue); in kq_initialize()
179 kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd, kq->queue->mqd_mem_obj); in kq_initialize()
181 uninit_queue(kq->queue); in kq_initialize()
183 kfd_gtt_sa_free(dev, kq->wptr_mem); in kq_initialize()
185 kfd_gtt_sa_free(dev, kq->rptr_mem); in kq_initialize()
187 kfd_gtt_sa_free(dev, kq->eop_mem); in kq_initialize()
189 kfd_gtt_sa_free(dev, kq->pq); in kq_initialize()
198 static void kq_uninitialize(struct kernel_queue *kq, bool hanging) in kq_uninitialize() argument
200 if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ && !hanging) in kq_uninitialize()
201 kq->mqd_mgr->destroy_mqd(kq->mqd_mgr, in kq_uninitialize()
202 kq->queue->mqd, in kq_uninitialize()
205 kq->queue->pipe, in kq_uninitialize()
206 kq->queue->queue); in kq_uninitialize()
207 else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ) in kq_uninitialize()
208 kfd_gtt_sa_free(kq->dev, kq->fence_mem_obj); in kq_uninitialize()
210 kq->mqd_mgr->free_mqd(kq->mqd_mgr, kq->queue->mqd, in kq_uninitialize()
211 kq->queue->mqd_mem_obj); in kq_uninitialize()
213 kfd_gtt_sa_free(kq->dev, kq->rptr_mem); in kq_uninitialize()
214 kfd_gtt_sa_free(kq->dev, kq->wptr_mem); in kq_uninitialize()
219 kfd_gtt_sa_free(kq->dev, kq->eop_mem); in kq_uninitialize()
221 kfd_gtt_sa_free(kq->dev, kq->pq); in kq_uninitialize()
222 kfd_release_kernel_doorbell(kq->dev, in kq_uninitialize()
223 kq->queue->properties.doorbell_ptr); in kq_uninitialize()
224 uninit_queue(kq->queue); in kq_uninitialize()
227 int kq_acquire_packet_buffer(struct kernel_queue *kq, in kq_acquire_packet_buffer() argument
241 rptr = *kq->rptr_kernel; in kq_acquire_packet_buffer()
242 wptr = kq->pending_wptr; in kq_acquire_packet_buffer()
243 wptr64 = kq->pending_wptr64; in kq_acquire_packet_buffer()
244 queue_address = (unsigned int *)kq->pq_kernel_addr; in kq_acquire_packet_buffer()
245 queue_size_dwords = kq->queue->properties.queue_size / 4; in kq_acquire_packet_buffer()
271 queue_address[wptr] = kq->nop_packet; in kq_acquire_packet_buffer()
278 kq->pending_wptr = wptr + packet_size_in_dwords; in kq_acquire_packet_buffer()
279 kq->pending_wptr64 = wptr64 + packet_size_in_dwords; in kq_acquire_packet_buffer()
288 void kq_submit_packet(struct kernel_queue *kq) in kq_submit_packet() argument
293 for (i = *kq->wptr_kernel; i < kq->pending_wptr; i++) { in kq_submit_packet()
294 pr_debug("0x%2X ", kq->pq_kernel_addr[i]); in kq_submit_packet()
300 if (kq->dev->device_info->doorbell_size == 8) { in kq_submit_packet()
301 *kq->wptr64_kernel = kq->pending_wptr64; in kq_submit_packet()
302 write_kernel_doorbell64(kq->queue->properties.doorbell_ptr, in kq_submit_packet()
303 kq->pending_wptr64); in kq_submit_packet()
305 *kq->wptr_kernel = kq->pending_wptr; in kq_submit_packet()
306 write_kernel_doorbell(kq->queue->properties.doorbell_ptr, in kq_submit_packet()
307 kq->pending_wptr); in kq_submit_packet()
311 void kq_rollback_packet(struct kernel_queue *kq) in kq_rollback_packet() argument
313 if (kq->dev->device_info->doorbell_size == 8) { in kq_rollback_packet()
314 kq->pending_wptr64 = *kq->wptr64_kernel; in kq_rollback_packet()
315 kq->pending_wptr = *kq->wptr_kernel % in kq_rollback_packet()
316 (kq->queue->properties.queue_size / 4); in kq_rollback_packet()
318 kq->pending_wptr = *kq->wptr_kernel; in kq_rollback_packet()
325 struct kernel_queue *kq; in kernel_queue_init() local
327 kq = kzalloc(sizeof(*kq), GFP_KERNEL); in kernel_queue_init()
328 if (!kq) in kernel_queue_init()
331 if (kq_initialize(kq, dev, type, KFD_KERNEL_QUEUE_SIZE)) in kernel_queue_init()
332 return kq; in kernel_queue_init()
336 kfree(kq); in kernel_queue_init()
340 void kernel_queue_uninit(struct kernel_queue *kq, bool hanging) in kernel_queue_uninit() argument
342 kq_uninitialize(kq, hanging); in kernel_queue_uninit()
343 kfree(kq); in kernel_queue_uninit()
349 struct kernel_queue *kq; in test_kq() local
355 kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_HIQ); in test_kq()
356 if (unlikely(!kq)) { in test_kq()
362 retval = kq_acquire_packet_buffer(kq, 5, &buffer); in test_kq()
369 buffer[i] = kq->nop_packet; in test_kq()
370 kq_submit_packet(kq); in test_kq()