Lines Matching refs:ring
67 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring) in radeon_fence_write() argument
69 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; in radeon_fence_write()
88 static u32 radeon_fence_read(struct radeon_device *rdev, int ring) in radeon_fence_read() argument
90 struct radeon_fence_driver *drv = &rdev->fence_drv[ring]; in radeon_fence_read()
113 static void radeon_fence_schedule_check(struct radeon_device *rdev, int ring) in radeon_fence_schedule_check() argument
120 &rdev->fence_drv[ring].lockup_work, in radeon_fence_schedule_check()
136 int ring) in radeon_fence_emit() argument
146 (*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring]; in radeon_fence_emit()
147 (*fence)->ring = ring; in radeon_fence_emit()
151 rdev->fence_context + ring, in radeon_fence_emit()
153 radeon_fence_ring_emit(rdev, ring, *fence); in radeon_fence_emit()
154 trace_radeon_fence_emit(rdev->ddev, ring, (*fence)->seq); in radeon_fence_emit()
155 radeon_fence_schedule_check(rdev, ring); in radeon_fence_emit()
177 seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq); in radeon_fence_check_signaled()
180 radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring); in radeon_fence_check_signaled()
197 static bool radeon_fence_activity(struct radeon_device *rdev, int ring) in radeon_fence_activity() argument
224 last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq); in radeon_fence_activity()
226 last_emitted = rdev->fence_drv[ring].sync_seq[ring]; in radeon_fence_activity()
227 seq = radeon_fence_read(rdev, ring); in radeon_fence_activity()
251 } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq); in radeon_fence_activity()
254 radeon_fence_schedule_check(rdev, ring); in radeon_fence_activity()
271 int ring; in radeon_fence_check_lockup() local
276 ring = fence_drv - &rdev->fence_drv[0]; in radeon_fence_check_lockup()
280 radeon_fence_schedule_check(rdev, ring); in radeon_fence_check_lockup()
293 if (radeon_fence_activity(rdev, ring)) in radeon_fence_check_lockup()
296 else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) { in radeon_fence_check_lockup()
302 fence_drv->sync_seq[ring], ring); in radeon_fence_check_lockup()
320 void radeon_fence_process(struct radeon_device *rdev, int ring) in radeon_fence_process() argument
322 if (radeon_fence_activity(rdev, ring)) in radeon_fence_process()
341 u64 seq, unsigned ring) in radeon_fence_seq_signaled() argument
343 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_seq_signaled()
347 radeon_fence_process(rdev, ring); in radeon_fence_seq_signaled()
348 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_seq_signaled()
358 unsigned ring = fence->ring; in radeon_fence_is_signaled() local
361 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_is_signaled()
366 radeon_fence_process(rdev, ring); in radeon_fence_is_signaled()
369 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) { in radeon_fence_is_signaled()
389 if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) in radeon_fence_enable_signaling()
393 radeon_irq_kms_sw_irq_get(rdev, fence->ring); in radeon_fence_enable_signaling()
395 if (radeon_fence_activity(rdev, fence->ring)) in radeon_fence_enable_signaling()
399 if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) { in radeon_fence_enable_signaling()
400 radeon_irq_kms_sw_irq_put(rdev, fence->ring); in radeon_fence_enable_signaling()
408 if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring)) in radeon_fence_enable_signaling()
409 rdev->fence_drv[fence->ring].delayed_irq = true; in radeon_fence_enable_signaling()
410 radeon_fence_schedule_check(rdev, fence->ring); in radeon_fence_enable_signaling()
434 if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { in radeon_fence_signaled()
550 seq[fence->ring] = fence->seq; in radeon_fence_wait_timeout()
634 int radeon_fence_wait_next(struct radeon_device *rdev, int ring) in radeon_fence_wait_next() argument
639 seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL; in radeon_fence_wait_next()
640 if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) { in radeon_fence_wait_next()
661 int radeon_fence_wait_empty(struct radeon_device *rdev, int ring) in radeon_fence_wait_empty() argument
666 seq[ring] = rdev->fence_drv[ring].sync_seq[ring]; in radeon_fence_wait_empty()
667 if (!seq[ring]) in radeon_fence_wait_empty()
676 ring, r); in radeon_fence_wait_empty()
722 unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring) in radeon_fence_count_emitted() argument
729 radeon_fence_process(rdev, ring); in radeon_fence_count_emitted()
730 emitted = rdev->fence_drv[ring].sync_seq[ring] in radeon_fence_count_emitted()
731 - atomic64_read(&rdev->fence_drv[ring].last_seq); in radeon_fence_count_emitted()
758 if (fence->ring == dst_ring) { in radeon_fence_need_sync()
764 if (fence->seq <= fdrv->sync_seq[fence->ring]) { in radeon_fence_need_sync()
789 if (fence->ring == dst_ring) { in radeon_fence_note_sync()
794 src = &fence->rdev->fence_drv[fence->ring]; in radeon_fence_note_sync()
816 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring) in radeon_fence_driver_start_ring() argument
821 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); in radeon_fence_driver_start_ring()
822 if (rdev->wb.use_event || !radeon_ring_supports_scratch_reg(rdev, &rdev->ring[ring])) { in radeon_fence_driver_start_ring()
823 rdev->fence_drv[ring].scratch_reg = 0; in radeon_fence_driver_start_ring()
824 if (ring != R600_RING_TYPE_UVD_INDEX) { in radeon_fence_driver_start_ring()
825 index = R600_WB_EVENT_OFFSET + ring * 4; in radeon_fence_driver_start_ring()
826 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; in radeon_fence_driver_start_ring()
827 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + in radeon_fence_driver_start_ring()
833 rdev->fence_drv[ring].cpu_addr = rdev->uvd.cpu_addr + index; in radeon_fence_driver_start_ring()
834 rdev->fence_drv[ring].gpu_addr = rdev->uvd.gpu_addr + index; in radeon_fence_driver_start_ring()
838 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg); in radeon_fence_driver_start_ring()
844 rdev->fence_drv[ring].scratch_reg - in radeon_fence_driver_start_ring()
846 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4]; in radeon_fence_driver_start_ring()
847 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index; in radeon_fence_driver_start_ring()
849 radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring); in radeon_fence_driver_start_ring()
850 rdev->fence_drv[ring].initialized = true; in radeon_fence_driver_start_ring()
852 ring, rdev->fence_drv[ring].gpu_addr); in radeon_fence_driver_start_ring()
866 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring) in radeon_fence_driver_init_ring() argument
870 rdev->fence_drv[ring].scratch_reg = -1; in radeon_fence_driver_init_ring()
871 rdev->fence_drv[ring].cpu_addr = NULL; in radeon_fence_driver_init_ring()
872 rdev->fence_drv[ring].gpu_addr = 0; in radeon_fence_driver_init_ring()
874 rdev->fence_drv[ring].sync_seq[i] = 0; in radeon_fence_driver_init_ring()
875 atomic64_set(&rdev->fence_drv[ring].last_seq, 0); in radeon_fence_driver_init_ring()
876 rdev->fence_drv[ring].initialized = false; in radeon_fence_driver_init_ring()
877 INIT_DELAYED_WORK(&rdev->fence_drv[ring].lockup_work, in radeon_fence_driver_init_ring()
879 rdev->fence_drv[ring].rdev = rdev; in radeon_fence_driver_init_ring()
895 int ring; in radeon_fence_driver_init() local
898 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { in radeon_fence_driver_init()
899 radeon_fence_driver_init_ring(rdev, ring); in radeon_fence_driver_init()
915 int ring, r; in radeon_fence_driver_fini() local
918 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) { in radeon_fence_driver_fini()
919 if (!rdev->fence_drv[ring].initialized) in radeon_fence_driver_fini()
921 r = radeon_fence_wait_empty(rdev, ring); in radeon_fence_driver_fini()
924 radeon_fence_driver_force_completion(rdev, ring); in radeon_fence_driver_fini()
926 cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); in radeon_fence_driver_fini()
928 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg); in radeon_fence_driver_fini()
929 rdev->fence_drv[ring].initialized = false; in radeon_fence_driver_fini()
943 void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring) in radeon_fence_driver_force_completion() argument
945 if (rdev->fence_drv[ring].initialized) { in radeon_fence_driver_force_completion()
946 radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring); in radeon_fence_driver_force_completion()
947 cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work); in radeon_fence_driver_force_completion()
1026 switch (fence->ring) { in radeon_fence_get_timeline_name()