Lines Matching refs:ring
61 int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw) in amdgpu_ring_alloc() argument
65 ndw = (ndw + ring->funcs->align_mask) & ~ring->funcs->align_mask; in amdgpu_ring_alloc()
70 if (WARN_ON_ONCE(ndw > ring->max_dw)) in amdgpu_ring_alloc()
73 ring->count_dw = ndw; in amdgpu_ring_alloc()
74 ring->wptr_old = ring->wptr; in amdgpu_ring_alloc()
76 if (ring->funcs->begin_use) in amdgpu_ring_alloc()
77 ring->funcs->begin_use(ring); in amdgpu_ring_alloc()
89 void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) in amdgpu_ring_insert_nop() argument
94 amdgpu_ring_write(ring, ring->funcs->nop); in amdgpu_ring_insert_nop()
105 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib) in amdgpu_ring_generic_pad_ib() argument
107 while (ib->length_dw & ring->funcs->align_mask) in amdgpu_ring_generic_pad_ib()
108 ib->ptr[ib->length_dw++] = ring->funcs->nop; in amdgpu_ring_generic_pad_ib()
120 void amdgpu_ring_commit(struct amdgpu_ring *ring) in amdgpu_ring_commit() argument
125 count = ring->funcs->align_mask + 1 - in amdgpu_ring_commit()
126 (ring->wptr & ring->funcs->align_mask); in amdgpu_ring_commit()
127 count %= ring->funcs->align_mask + 1; in amdgpu_ring_commit()
128 ring->funcs->insert_nop(ring, count); in amdgpu_ring_commit()
131 amdgpu_ring_set_wptr(ring); in amdgpu_ring_commit()
133 if (ring->funcs->end_use) in amdgpu_ring_commit()
134 ring->funcs->end_use(ring); in amdgpu_ring_commit()
144 void amdgpu_ring_undo(struct amdgpu_ring *ring) in amdgpu_ring_undo() argument
146 ring->wptr = ring->wptr_old; in amdgpu_ring_undo()
148 if (ring->funcs->end_use) in amdgpu_ring_undo()
149 ring->funcs->end_use(ring); in amdgpu_ring_undo()
166 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, in amdgpu_ring_init() argument
182 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) in amdgpu_ring_init()
184 else if (ring == &adev->sdma.instance[0].page) in amdgpu_ring_init()
187 if (ring->adev == NULL) { in amdgpu_ring_init()
191 ring->adev = adev; in amdgpu_ring_init()
192 ring->idx = adev->num_rings++; in amdgpu_ring_init()
193 adev->rings[ring->idx] = ring; in amdgpu_ring_init()
194 r = amdgpu_fence_driver_init_ring(ring, sched_hw_submission, in amdgpu_ring_init()
200 r = amdgpu_device_wb_get(adev, &ring->rptr_offs); in amdgpu_ring_init()
206 r = amdgpu_device_wb_get(adev, &ring->wptr_offs); in amdgpu_ring_init()
212 r = amdgpu_device_wb_get(adev, &ring->fence_offs); in amdgpu_ring_init()
218 r = amdgpu_device_wb_get(adev, &ring->trail_fence_offs); in amdgpu_ring_init()
224 ring->trail_fence_gpu_addr = in amdgpu_ring_init()
225 adev->wb.gpu_addr + (ring->trail_fence_offs * 4); in amdgpu_ring_init()
226 ring->trail_fence_cpu_addr = &adev->wb.wb[ring->trail_fence_offs]; in amdgpu_ring_init()
228 r = amdgpu_device_wb_get(adev, &ring->cond_exe_offs); in amdgpu_ring_init()
233 ring->cond_exe_gpu_addr = adev->wb.gpu_addr + (ring->cond_exe_offs * 4); in amdgpu_ring_init()
234 ring->cond_exe_cpu_addr = &adev->wb.wb[ring->cond_exe_offs]; in amdgpu_ring_init()
236 *ring->cond_exe_cpu_addr = 1; in amdgpu_ring_init()
238 r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type); in amdgpu_ring_init()
244 ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission); in amdgpu_ring_init()
246 ring->buf_mask = (ring->ring_size / 4) - 1; in amdgpu_ring_init()
247 ring->ptr_mask = ring->funcs->support_64bit_ptrs ? in amdgpu_ring_init()
248 0xffffffffffffffff : ring->buf_mask; in amdgpu_ring_init()
250 if (ring->ring_obj == NULL) { in amdgpu_ring_init()
251 r = amdgpu_bo_create_kernel(adev, ring->ring_size + ring->funcs->extra_dw, PAGE_SIZE, in amdgpu_ring_init()
253 &ring->ring_obj, in amdgpu_ring_init()
254 &ring->gpu_addr, in amdgpu_ring_init()
255 (void **)&ring->ring); in amdgpu_ring_init()
260 amdgpu_ring_clear_ring(ring); in amdgpu_ring_init()
263 ring->max_dw = max_dw; in amdgpu_ring_init()
264 ring->hw_prio = hw_prio; in amdgpu_ring_init()
266 if (!ring->no_scheduler) { in amdgpu_ring_init()
267 hw_ip = ring->funcs->type; in amdgpu_ring_init()
270 &ring->sched; in amdgpu_ring_init()
283 void amdgpu_ring_fini(struct amdgpu_ring *ring) in amdgpu_ring_fini() argument
287 if (!(ring->adev) || !(ring->adev->rings[ring->idx])) in amdgpu_ring_fini()
290 ring->sched.ready = false; in amdgpu_ring_fini()
292 amdgpu_device_wb_free(ring->adev, ring->rptr_offs); in amdgpu_ring_fini()
293 amdgpu_device_wb_free(ring->adev, ring->wptr_offs); in amdgpu_ring_fini()
295 amdgpu_device_wb_free(ring->adev, ring->cond_exe_offs); in amdgpu_ring_fini()
296 amdgpu_device_wb_free(ring->adev, ring->fence_offs); in amdgpu_ring_fini()
298 amdgpu_bo_free_kernel(&ring->ring_obj, in amdgpu_ring_fini()
299 &ring->gpu_addr, in amdgpu_ring_fini()
300 (void **)&ring->ring); in amdgpu_ring_fini()
302 dma_fence_put(ring->vmid_wait); in amdgpu_ring_fini()
303 ring->vmid_wait = NULL; in amdgpu_ring_fini()
304 ring->me = 0; in amdgpu_ring_fini()
306 ring->adev->rings[ring->idx] = NULL; in amdgpu_ring_fini()
321 void amdgpu_ring_emit_reg_write_reg_wait_helper(struct amdgpu_ring *ring, in amdgpu_ring_emit_reg_write_reg_wait_helper() argument
325 amdgpu_ring_emit_wreg(ring, reg0, ref); in amdgpu_ring_emit_reg_write_reg_wait_helper()
326 amdgpu_ring_emit_reg_wait(ring, reg1, mask, mask); in amdgpu_ring_emit_reg_write_reg_wait_helper()
338 bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, in amdgpu_ring_soft_recovery() argument
343 if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence) in amdgpu_ring_soft_recovery()
346 atomic_inc(&ring->adev->gpu_reset_counter); in amdgpu_ring_soft_recovery()
349 ring->funcs->soft_recovery(ring, vmid); in amdgpu_ring_soft_recovery()
369 struct amdgpu_ring *ring = file_inode(f)->i_private; in amdgpu_debugfs_ring_read() local
379 early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask; in amdgpu_debugfs_ring_read()
380 early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask; in amdgpu_debugfs_ring_read()
381 early[2] = ring->wptr & ring->buf_mask; in amdgpu_debugfs_ring_read()
394 if (*pos >= (ring->ring_size + 12)) in amdgpu_debugfs_ring_read()
397 value = ring->ring[(*pos - 12)/4]; in amdgpu_debugfs_ring_read()
419 struct amdgpu_ring *ring) in amdgpu_debugfs_ring_init() argument
426 sprintf(name, "amdgpu_ring_%s", ring->name); in amdgpu_debugfs_ring_init()
427 debugfs_create_file_size(name, S_IFREG | S_IRUGO, root, ring, in amdgpu_debugfs_ring_init()
429 ring->ring_size + 12); in amdgpu_debugfs_ring_init()
443 int amdgpu_ring_test_helper(struct amdgpu_ring *ring) in amdgpu_ring_test_helper() argument
445 struct amdgpu_device *adev = ring->adev; in amdgpu_ring_test_helper()
448 r = amdgpu_ring_test_ring(ring); in amdgpu_ring_test_helper()
451 ring->name, r); in amdgpu_ring_test_helper()
454 ring->name); in amdgpu_ring_test_helper()
456 ring->sched.ready = !r; in amdgpu_ring_test_helper()