Lines Matching refs:pfdev
87 static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num) in panfrost_fence_create() argument
90 struct panfrost_job_slot *js = pfdev->js; in panfrost_fence_create()
96 fence->dev = pfdev->ddev; in panfrost_fence_create()
118 (job->pfdev->features.nr_core_groups == 2)) in panfrost_job_get_slot()
120 if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987)) in panfrost_job_get_slot()
127 static void panfrost_job_write_affinity(struct panfrost_device *pfdev, in panfrost_job_write_affinity() argument
138 affinity = pfdev->features.shader_present; in panfrost_job_write_affinity()
140 job_write(pfdev, JS_AFFINITY_NEXT_LO(js), lower_32_bits(affinity)); in panfrost_job_write_affinity()
141 job_write(pfdev, JS_AFFINITY_NEXT_HI(js), upper_32_bits(affinity)); in panfrost_job_write_affinity()
149 if (!panfrost_has_hw_feature(job->pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) in panfrost_get_job_chain_flag()
156 panfrost_dequeue_job(struct panfrost_device *pfdev, int slot) in panfrost_dequeue_job() argument
158 struct panfrost_job *job = pfdev->jobs[slot][0]; in panfrost_dequeue_job()
161 pfdev->jobs[slot][0] = pfdev->jobs[slot][1]; in panfrost_dequeue_job()
162 pfdev->jobs[slot][1] = NULL; in panfrost_dequeue_job()
168 panfrost_enqueue_job(struct panfrost_device *pfdev, int slot, in panfrost_enqueue_job() argument
174 if (!pfdev->jobs[slot][0]) { in panfrost_enqueue_job()
175 pfdev->jobs[slot][0] = job; in panfrost_enqueue_job()
179 WARN_ON(pfdev->jobs[slot][1]); in panfrost_enqueue_job()
180 pfdev->jobs[slot][1] = job; in panfrost_enqueue_job()
182 panfrost_get_job_chain_flag(pfdev->jobs[slot][0])); in panfrost_enqueue_job()
188 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_hw_submit() local
194 panfrost_devfreq_record_busy(&pfdev->pfdevfreq); in panfrost_job_hw_submit()
196 ret = pm_runtime_get_sync(pfdev->dev); in panfrost_job_hw_submit()
200 if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) { in panfrost_job_hw_submit()
204 cfg = panfrost_mmu_as_get(pfdev, job->file_priv->mmu); in panfrost_job_hw_submit()
206 job_write(pfdev, JS_HEAD_NEXT_LO(js), lower_32_bits(jc_head)); in panfrost_job_hw_submit()
207 job_write(pfdev, JS_HEAD_NEXT_HI(js), upper_32_bits(jc_head)); in panfrost_job_hw_submit()
209 panfrost_job_write_affinity(pfdev, job->requirements, js); in panfrost_job_hw_submit()
218 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION)) in panfrost_job_hw_submit()
221 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649)) in panfrost_job_hw_submit()
224 job_write(pfdev, JS_CONFIG_NEXT(js), cfg); in panfrost_job_hw_submit()
226 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION)) in panfrost_job_hw_submit()
227 job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id); in panfrost_job_hw_submit()
231 spin_lock(&pfdev->js->job_lock); in panfrost_job_hw_submit()
232 subslot = panfrost_enqueue_job(pfdev, js, job); in panfrost_job_hw_submit()
234 if (!atomic_read(&pfdev->reset.pending)) { in panfrost_job_hw_submit()
235 job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START); in panfrost_job_hw_submit()
236 dev_dbg(pfdev->dev, in panfrost_job_hw_submit()
240 spin_unlock(&pfdev->js->job_lock); in panfrost_job_hw_submit()
272 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_push() local
281 mutex_lock(&pfdev->sched_lock); in panfrost_job_push()
289 mutex_unlock(&pfdev->sched_lock); in panfrost_job_push()
297 mutex_unlock(&pfdev->sched_lock); in panfrost_job_push()
355 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_run() local
368 fence = panfrost_fence_create(pfdev, slot); in panfrost_job_run()
381 void panfrost_job_enable_interrupts(struct panfrost_device *pfdev) in panfrost_job_enable_interrupts() argument
390 job_write(pfdev, JOB_INT_CLEAR, irq_mask); in panfrost_job_enable_interrupts()
391 job_write(pfdev, JOB_INT_MASK, irq_mask); in panfrost_job_enable_interrupts()
394 static void panfrost_job_handle_err(struct panfrost_device *pfdev, in panfrost_job_handle_err() argument
398 u32 js_status = job_read(pfdev, JS_STATUS(js)); in panfrost_job_handle_err()
403 dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x", in panfrost_job_handle_err()
405 job_read(pfdev, JS_HEAD_LO(js)), in panfrost_job_handle_err()
406 job_read(pfdev, JS_TAIL_LO(js))); in panfrost_job_handle_err()
408 dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x", in panfrost_job_handle_err()
410 job_read(pfdev, JS_HEAD_LO(js)), in panfrost_job_handle_err()
411 job_read(pfdev, JS_TAIL_LO(js))); in panfrost_job_handle_err()
416 job->jc = job_read(pfdev, JS_TAIL_LO(js)) | in panfrost_job_handle_err()
417 ((u64)job_read(pfdev, JS_TAIL_HI(js)) << 32); in panfrost_job_handle_err()
434 panfrost_mmu_as_put(pfdev, job->file_priv->mmu); in panfrost_job_handle_err()
435 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_job_handle_err()
440 pm_runtime_put_autosuspend(pfdev->dev); in panfrost_job_handle_err()
442 if (panfrost_exception_needs_reset(pfdev, js_status)) { in panfrost_job_handle_err()
443 atomic_set(&pfdev->reset.pending, 1); in panfrost_job_handle_err()
444 drm_sched_fault(&pfdev->js->queue[js].sched); in panfrost_job_handle_err()
448 static void panfrost_job_handle_done(struct panfrost_device *pfdev, in panfrost_job_handle_done() argument
455 panfrost_mmu_as_put(pfdev, job->file_priv->mmu); in panfrost_job_handle_done()
456 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_job_handle_done()
459 pm_runtime_put_autosuspend(pfdev->dev); in panfrost_job_handle_done()
462 static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status) in panfrost_job_handle_irq() argument
479 done[j][1] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
481 done[j][0] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
489 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP); in panfrost_job_handle_irq()
490 failed[j] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
506 job_write(pfdev, JOB_INT_CLEAR, status); in panfrost_job_handle_irq()
508 js_state |= job_read(pfdev, JOB_INT_JS_STATE) & js_state_mask; in panfrost_job_handle_irq()
510 status = job_read(pfdev, JOB_INT_RAWSTAT); in panfrost_job_handle_irq()
519 panfrost_job_handle_err(pfdev, failed[j], j); in panfrost_job_handle_irq()
520 } else if (pfdev->jobs[j][0] && !(js_state & MK_JS_MASK(j))) { in panfrost_job_handle_irq()
531 done[j][0] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
533 done[j][1] = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
537 panfrost_job_handle_done(pfdev, done[j][i]); in panfrost_job_handle_irq()
547 if (!failed[j] || !pfdev->jobs[j][0]) in panfrost_job_handle_irq()
550 if (pfdev->jobs[j][0]->jc == 0) { in panfrost_job_handle_irq()
552 struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j); in panfrost_job_handle_irq()
555 panfrost_job_handle_done(pfdev, canceled); in panfrost_job_handle_irq()
556 } else if (!atomic_read(&pfdev->reset.pending)) { in panfrost_job_handle_irq()
558 job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START); in panfrost_job_handle_irq()
563 static void panfrost_job_handle_irqs(struct panfrost_device *pfdev) in panfrost_job_handle_irqs() argument
565 u32 status = job_read(pfdev, JOB_INT_RAWSTAT); in panfrost_job_handle_irqs()
568 pm_runtime_mark_last_busy(pfdev->dev); in panfrost_job_handle_irqs()
570 spin_lock(&pfdev->js->job_lock); in panfrost_job_handle_irqs()
571 panfrost_job_handle_irq(pfdev, status); in panfrost_job_handle_irqs()
572 spin_unlock(&pfdev->js->job_lock); in panfrost_job_handle_irqs()
573 status = job_read(pfdev, JOB_INT_RAWSTAT); in panfrost_job_handle_irqs()
577 static u32 panfrost_active_slots(struct panfrost_device *pfdev, in panfrost_active_slots() argument
585 rawstat = job_read(pfdev, JOB_INT_RAWSTAT); in panfrost_active_slots()
599 panfrost_reset(struct panfrost_device *pfdev, in panfrost_reset() argument
607 if (!atomic_read(&pfdev->reset.pending)) in panfrost_reset()
624 drm_sched_stop(&pfdev->js->queue[i].sched, bad); in panfrost_reset()
634 job_write(pfdev, JOB_INT_MASK, 0); in panfrost_reset()
635 synchronize_irq(pfdev->js->irq); in panfrost_reset()
639 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP); in panfrost_reset()
640 job_write(pfdev, JS_COMMAND(i), JS_COMMAND_SOFT_STOP); in panfrost_reset()
644 ret = readl_poll_timeout(pfdev->iomem + JOB_INT_JS_STATE, js_state, in panfrost_reset()
645 !panfrost_active_slots(pfdev, &js_state_mask, js_state), in panfrost_reset()
649 dev_err(pfdev->dev, "Soft-stop failed\n"); in panfrost_reset()
652 panfrost_job_handle_irqs(pfdev); in panfrost_reset()
659 spin_lock(&pfdev->js->job_lock); in panfrost_reset()
661 for (j = 0; j < ARRAY_SIZE(pfdev->jobs[0]) && pfdev->jobs[i][j]; j++) { in panfrost_reset()
662 pm_runtime_put_noidle(pfdev->dev); in panfrost_reset()
663 panfrost_devfreq_record_idle(&pfdev->pfdevfreq); in panfrost_reset()
666 memset(pfdev->jobs, 0, sizeof(pfdev->jobs)); in panfrost_reset()
667 spin_unlock(&pfdev->js->job_lock); in panfrost_reset()
670 panfrost_device_reset(pfdev); in panfrost_reset()
675 job_write(pfdev, JOB_INT_MASK, 0); in panfrost_reset()
678 atomic_set(&pfdev->reset.pending, 0); in panfrost_reset()
689 drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched); in panfrost_reset()
694 drm_sched_start(&pfdev->js->queue[i].sched, true); in panfrost_reset()
697 job_write(pfdev, JOB_INT_MASK, in panfrost_reset()
708 struct panfrost_device *pfdev = job->pfdev; in panfrost_job_timedout() local
718 …dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sch… in panfrost_job_timedout()
720 job_read(pfdev, JS_CONFIG(js)), in panfrost_job_timedout()
721 job_read(pfdev, JS_STATUS(js)), in panfrost_job_timedout()
722 job_read(pfdev, JS_HEAD_LO(js)), in panfrost_job_timedout()
723 job_read(pfdev, JS_TAIL_LO(js)), in panfrost_job_timedout()
726 atomic_set(&pfdev->reset.pending, 1); in panfrost_job_timedout()
727 panfrost_reset(pfdev, sched_job); in panfrost_job_timedout()
734 struct panfrost_device *pfdev; in panfrost_reset_work() local
736 pfdev = container_of(work, struct panfrost_device, reset.work); in panfrost_reset_work()
737 panfrost_reset(pfdev, NULL); in panfrost_reset_work()
748 struct panfrost_device *pfdev = data; in panfrost_job_irq_handler_thread() local
750 panfrost_job_handle_irqs(pfdev); in panfrost_job_irq_handler_thread()
751 job_write(pfdev, JOB_INT_MASK, in panfrost_job_irq_handler_thread()
759 struct panfrost_device *pfdev = data; in panfrost_job_irq_handler() local
760 u32 status = job_read(pfdev, JOB_INT_STAT); in panfrost_job_irq_handler()
765 job_write(pfdev, JOB_INT_MASK, 0); in panfrost_job_irq_handler()
769 int panfrost_job_init(struct panfrost_device *pfdev) in panfrost_job_init() argument
779 if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) in panfrost_job_init()
782 pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL); in panfrost_job_init()
786 INIT_WORK(&pfdev->reset.work, panfrost_reset_work); in panfrost_job_init()
789 js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job"); in panfrost_job_init()
793 ret = devm_request_threaded_irq(pfdev->dev, js->irq, in panfrost_job_init()
797 pfdev); in panfrost_job_init()
799 dev_err(pfdev->dev, "failed to request job irq"); in panfrost_job_init()
803 pfdev->reset.wq = alloc_ordered_workqueue("panfrost-reset", 0); in panfrost_job_init()
804 if (!pfdev->reset.wq) in panfrost_job_init()
814 pfdev->reset.wq, in panfrost_job_init()
817 dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret); in panfrost_job_init()
822 panfrost_job_enable_interrupts(pfdev); in panfrost_job_init()
830 destroy_workqueue(pfdev->reset.wq); in panfrost_job_init()
834 void panfrost_job_fini(struct panfrost_device *pfdev) in panfrost_job_fini() argument
836 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_fini()
839 job_write(pfdev, JOB_INT_MASK, 0); in panfrost_job_fini()
845 cancel_work_sync(&pfdev->reset.work); in panfrost_job_fini()
846 destroy_workqueue(pfdev->reset.wq); in panfrost_job_fini()
851 struct panfrost_device *pfdev = panfrost_priv->pfdev; in panfrost_job_open() local
852 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_open()
869 struct panfrost_device *pfdev = panfrost_priv->pfdev; in panfrost_job_close() local
876 spin_lock(&pfdev->js->job_lock); in panfrost_job_close()
881 for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) { in panfrost_job_close()
882 struct panfrost_job *job = pfdev->jobs[i][j]; in panfrost_job_close()
890 job_write(pfdev, JS_COMMAND_NEXT(i), JS_COMMAND_NOP); in panfrost_job_close()
897 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) { in panfrost_job_close()
905 job_write(pfdev, JS_COMMAND(i), cmd); in panfrost_job_close()
908 spin_unlock(&pfdev->js->job_lock); in panfrost_job_close()
911 int panfrost_job_is_idle(struct panfrost_device *pfdev) in panfrost_job_is_idle() argument
913 struct panfrost_job_slot *js = pfdev->js; in panfrost_job_is_idle()