Lines Matching refs:cs
15 u32 *cs, flags = 0; in gen8_emit_flush_rcs() local
57 cs = intel_ring_begin(rq, len); in gen8_emit_flush_rcs()
58 if (IS_ERR(cs)) in gen8_emit_flush_rcs()
59 return PTR_ERR(cs); in gen8_emit_flush_rcs()
62 cs = gen8_emit_pipe_control(cs, 0, 0); in gen8_emit_flush_rcs()
65 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE, in gen8_emit_flush_rcs()
68 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen8_emit_flush_rcs()
71 cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0); in gen8_emit_flush_rcs()
73 intel_ring_advance(rq, cs); in gen8_emit_flush_rcs()
80 u32 cmd, *cs; in gen8_emit_flush_xcs() local
82 cs = intel_ring_begin(rq, 4); in gen8_emit_flush_xcs()
83 if (IS_ERR(cs)) in gen8_emit_flush_xcs()
84 return PTR_ERR(cs); in gen8_emit_flush_xcs()
102 *cs++ = cmd; in gen8_emit_flush_xcs()
103 *cs++ = LRC_PPHWSP_SCRATCH_ADDR; in gen8_emit_flush_xcs()
104 *cs++ = 0; /* upper addr */ in gen8_emit_flush_xcs()
105 *cs++ = 0; /* value */ in gen8_emit_flush_xcs()
106 intel_ring_advance(rq, cs); in gen8_emit_flush_xcs()
114 u32 *cs; in gen11_emit_flush_rcs() local
127 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
128 if (IS_ERR(cs)) in gen11_emit_flush_rcs()
129 return PTR_ERR(cs); in gen11_emit_flush_rcs()
131 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen11_emit_flush_rcs()
132 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
136 u32 *cs; in gen11_emit_flush_rcs() local
151 cs = intel_ring_begin(rq, 6); in gen11_emit_flush_rcs()
152 if (IS_ERR(cs)) in gen11_emit_flush_rcs()
153 return PTR_ERR(cs); in gen11_emit_flush_rcs()
155 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen11_emit_flush_rcs()
156 intel_ring_advance(rq, cs); in gen11_emit_flush_rcs()
191 static u32 *gen12_emit_aux_table_inv(const i915_reg_t inv_reg, u32 *cs) in gen12_emit_aux_table_inv() argument
193 *cs++ = MI_LOAD_REGISTER_IMM(1); in gen12_emit_aux_table_inv()
194 *cs++ = i915_mmio_reg_offset(inv_reg); in gen12_emit_aux_table_inv()
195 *cs++ = AUX_INV; in gen12_emit_aux_table_inv()
196 *cs++ = MI_NOOP; in gen12_emit_aux_table_inv()
198 return cs; in gen12_emit_aux_table_inv()
205 u32 *cs; in gen12_emit_flush_rcs() local
221 cs = intel_ring_begin(rq, 6); in gen12_emit_flush_rcs()
222 if (IS_ERR(cs)) in gen12_emit_flush_rcs()
223 return PTR_ERR(cs); in gen12_emit_flush_rcs()
225 cs = gen12_emit_pipe_control(cs, in gen12_emit_flush_rcs()
228 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
233 u32 *cs; in gen12_emit_flush_rcs() local
248 cs = intel_ring_begin(rq, 8 + 4); in gen12_emit_flush_rcs()
249 if (IS_ERR(cs)) in gen12_emit_flush_rcs()
250 return PTR_ERR(cs); in gen12_emit_flush_rcs()
257 *cs++ = preparser_disable(true); in gen12_emit_flush_rcs()
259 cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); in gen12_emit_flush_rcs()
262 cs = gen12_emit_aux_table_inv(GEN12_GFX_CCS_AUX_NV, cs); in gen12_emit_flush_rcs()
264 *cs++ = preparser_disable(false); in gen12_emit_flush_rcs()
265 intel_ring_advance(rq, cs); in gen12_emit_flush_rcs()
274 u32 cmd, *cs; in gen12_emit_flush_xcs() local
284 cs = intel_ring_begin(rq, cmd); in gen12_emit_flush_xcs()
285 if (IS_ERR(cs)) in gen12_emit_flush_xcs()
286 return PTR_ERR(cs); in gen12_emit_flush_xcs()
289 *cs++ = preparser_disable(true); in gen12_emit_flush_xcs()
307 *cs++ = cmd; in gen12_emit_flush_xcs()
308 *cs++ = LRC_PPHWSP_SCRATCH_ADDR; in gen12_emit_flush_xcs()
309 *cs++ = 0; /* upper addr */ in gen12_emit_flush_xcs()
310 *cs++ = 0; /* value */ in gen12_emit_flush_xcs()
316 *cs++ = MI_LOAD_REGISTER_IMM(hweight32(aux_inv)); in gen12_emit_flush_xcs()
318 *cs++ = i915_mmio_reg_offset(aux_inv_reg(engine)); in gen12_emit_flush_xcs()
319 *cs++ = AUX_INV; in gen12_emit_flush_xcs()
321 *cs++ = MI_NOOP; in gen12_emit_flush_xcs()
325 *cs++ = preparser_disable(false); in gen12_emit_flush_xcs()
327 intel_ring_advance(rq, cs); in gen12_emit_flush_xcs()
352 u32 *cs; in gen8_emit_init_breadcrumb() local
358 cs = intel_ring_begin(rq, 6); in gen8_emit_init_breadcrumb()
359 if (IS_ERR(cs)) in gen8_emit_init_breadcrumb()
360 return PTR_ERR(cs); in gen8_emit_init_breadcrumb()
362 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; in gen8_emit_init_breadcrumb()
363 *cs++ = hwsp_offset(rq); in gen8_emit_init_breadcrumb()
364 *cs++ = 0; in gen8_emit_init_breadcrumb()
365 *cs++ = rq->fence.seqno - 1; in gen8_emit_init_breadcrumb()
384 *cs++ = MI_NOOP; in gen8_emit_init_breadcrumb()
385 *cs++ = MI_ARB_CHECK; in gen8_emit_init_breadcrumb()
387 intel_ring_advance(rq, cs); in gen8_emit_init_breadcrumb()
390 rq->infix = intel_ring_offset(rq, cs); in gen8_emit_init_breadcrumb()
401 u32 *cs; in gen8_emit_bb_start_noarb() local
403 cs = intel_ring_begin(rq, 4); in gen8_emit_bb_start_noarb()
404 if (IS_ERR(cs)) in gen8_emit_bb_start_noarb()
405 return PTR_ERR(cs); in gen8_emit_bb_start_noarb()
420 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in gen8_emit_bb_start_noarb()
423 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in gen8_emit_bb_start_noarb()
425 *cs++ = lower_32_bits(offset); in gen8_emit_bb_start_noarb()
426 *cs++ = upper_32_bits(offset); in gen8_emit_bb_start_noarb()
428 intel_ring_advance(rq, cs); in gen8_emit_bb_start_noarb()
437 u32 *cs; in gen8_emit_bb_start() local
442 cs = intel_ring_begin(rq, 6); in gen8_emit_bb_start()
443 if (IS_ERR(cs)) in gen8_emit_bb_start()
444 return PTR_ERR(cs); in gen8_emit_bb_start()
446 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen8_emit_bb_start()
448 *cs++ = MI_BATCH_BUFFER_START_GEN8 | in gen8_emit_bb_start()
450 *cs++ = lower_32_bits(offset); in gen8_emit_bb_start()
451 *cs++ = upper_32_bits(offset); in gen8_emit_bb_start()
453 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; in gen8_emit_bb_start()
454 *cs++ = MI_NOOP; in gen8_emit_bb_start()
456 intel_ring_advance(rq, cs); in gen8_emit_bb_start()
474 static u32 *gen8_emit_wa_tail(struct i915_request *rq, u32 *cs) in gen8_emit_wa_tail() argument
477 *cs++ = MI_ARB_CHECK; in gen8_emit_wa_tail()
478 *cs++ = MI_NOOP; in gen8_emit_wa_tail()
479 rq->wa_tail = intel_ring_offset(rq, cs); in gen8_emit_wa_tail()
484 return cs; in gen8_emit_wa_tail()
487 static u32 *emit_preempt_busywait(struct i915_request *rq, u32 *cs) in emit_preempt_busywait() argument
489 *cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */ in emit_preempt_busywait()
490 *cs++ = MI_SEMAPHORE_WAIT | in emit_preempt_busywait()
494 *cs++ = 0; in emit_preempt_busywait()
495 *cs++ = preempt_address(rq->engine); in emit_preempt_busywait()
496 *cs++ = 0; in emit_preempt_busywait()
497 *cs++ = MI_NOOP; in emit_preempt_busywait()
499 return cs; in emit_preempt_busywait()
503 gen8_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_tail() argument
505 *cs++ = MI_USER_INTERRUPT; in gen8_emit_fini_breadcrumb_tail()
507 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen8_emit_fini_breadcrumb_tail()
510 cs = emit_preempt_busywait(rq, cs); in gen8_emit_fini_breadcrumb_tail()
512 rq->tail = intel_ring_offset(rq, cs); in gen8_emit_fini_breadcrumb_tail()
515 return gen8_emit_wa_tail(rq, cs); in gen8_emit_fini_breadcrumb_tail()
518 static u32 *emit_xcs_breadcrumb(struct i915_request *rq, u32 *cs) in emit_xcs_breadcrumb() argument
520 return gen8_emit_ggtt_write(cs, rq->fence.seqno, hwsp_offset(rq), 0); in emit_xcs_breadcrumb()
523 u32 *gen8_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_xcs() argument
525 return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs)); in gen8_emit_fini_breadcrumb_xcs()
528 u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen8_emit_fini_breadcrumb_rcs() argument
530 cs = gen8_emit_pipe_control(cs, in gen8_emit_fini_breadcrumb_rcs()
537 cs = gen8_emit_ggtt_write_rcs(cs, in gen8_emit_fini_breadcrumb_rcs()
543 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen8_emit_fini_breadcrumb_rcs()
546 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen11_emit_fini_breadcrumb_rcs() argument
548 cs = gen8_emit_ggtt_write_rcs(cs, in gen11_emit_fini_breadcrumb_rcs()
558 return gen8_emit_fini_breadcrumb_tail(rq, cs); in gen11_emit_fini_breadcrumb_rcs()
580 static u32 *gen12_emit_preempt_busywait(struct i915_request *rq, u32 *cs) in gen12_emit_preempt_busywait() argument
582 *cs++ = MI_ARB_CHECK; /* trigger IDLE->ACTIVE first */ in gen12_emit_preempt_busywait()
583 *cs++ = MI_SEMAPHORE_WAIT_TOKEN | in gen12_emit_preempt_busywait()
587 *cs++ = 0; in gen12_emit_preempt_busywait()
588 *cs++ = preempt_address(rq->engine); in gen12_emit_preempt_busywait()
589 *cs++ = 0; in gen12_emit_preempt_busywait()
590 *cs++ = 0; in gen12_emit_preempt_busywait()
592 return cs; in gen12_emit_preempt_busywait()
596 gen12_emit_fini_breadcrumb_tail(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_tail() argument
598 *cs++ = MI_USER_INTERRUPT; in gen12_emit_fini_breadcrumb_tail()
600 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; in gen12_emit_fini_breadcrumb_tail()
603 cs = gen12_emit_preempt_busywait(rq, cs); in gen12_emit_fini_breadcrumb_tail()
605 rq->tail = intel_ring_offset(rq, cs); in gen12_emit_fini_breadcrumb_tail()
608 return gen8_emit_wa_tail(rq, cs); in gen12_emit_fini_breadcrumb_tail()
611 u32 *gen12_emit_fini_breadcrumb_xcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_xcs() argument
614 cs = emit_xcs_breadcrumb(rq, __gen8_emit_flush_dw(cs, 0, 0, 0)); in gen12_emit_fini_breadcrumb_xcs()
615 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_xcs()
618 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs) in gen12_emit_fini_breadcrumb_rcs() argument
620 cs = gen12_emit_ggtt_write_rcs(cs, in gen12_emit_fini_breadcrumb_rcs()
634 return gen12_emit_fini_breadcrumb_tail(rq, cs); in gen12_emit_fini_breadcrumb_rcs()