Lines Matching refs:wal
55 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name) in wa_init_start() argument
57 wal->name = name; in wa_init_start()
58 wal->engine_name = engine_name; in wa_init_start()
63 static void wa_init_finish(struct i915_wa_list *wal) in wa_init_finish() argument
66 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) { in wa_init_finish()
67 struct i915_wa *list = kmemdup(wal->list, in wa_init_finish()
68 wal->count * sizeof(*list), in wa_init_finish()
72 kfree(wal->list); in wa_init_finish()
73 wal->list = list; in wa_init_finish()
77 if (!wal->count) in wa_init_finish()
81 wal->wa_count, wal->name, wal->engine_name); in wa_init_finish()
84 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) in _wa_add() argument
87 unsigned int start = 0, end = wal->count; in _wa_add()
93 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */ in _wa_add()
96 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa), in _wa_add()
103 if (wal->list) { in _wa_add()
104 memcpy(list, wal->list, sizeof(*wa) * wal->count); in _wa_add()
105 kfree(wal->list); in _wa_add()
108 wal->list = list; in _wa_add()
114 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) { in _wa_add()
116 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) { in _wa_add()
119 wa_ = &wal->list[mid]; in _wa_add()
129 wal->wa_count++; in _wa_add()
137 wal->wa_count++; in _wa_add()
138 wa_ = &wal->list[wal->count++]; in _wa_add()
141 while (wa_-- > wal->list) { in _wa_add()
152 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg, in wa_add() argument
163 _wa_add(wal, &wa); in wa_add()
167 wa_write_clr_set(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set) in wa_write_clr_set() argument
169 wa_add(wal, reg, clear, set, clear, false); in wa_write_clr_set()
173 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set) in wa_write() argument
175 wa_write_clr_set(wal, reg, ~0, set); in wa_write()
179 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set) in wa_write_or() argument
181 wa_write_clr_set(wal, reg, set, set); in wa_write_or()
185 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr) in wa_write_clr() argument
187 wa_write_clr_set(wal, reg, clr, 0); in wa_write_clr()
202 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val) in wa_masked_en() argument
204 wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val, true); in wa_masked_en()
208 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val) in wa_masked_dis() argument
210 wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val, true); in wa_masked_dis()
214 wa_masked_field_set(struct i915_wa_list *wal, i915_reg_t reg, in wa_masked_field_set() argument
217 wa_add(wal, reg, 0, _MASKED_FIELD(mask, val), mask, true); in wa_masked_field_set()
221 struct i915_wa_list *wal) in gen6_ctx_workarounds_init() argument
223 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); in gen6_ctx_workarounds_init()
227 struct i915_wa_list *wal) in gen7_ctx_workarounds_init() argument
229 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); in gen7_ctx_workarounds_init()
233 struct i915_wa_list *wal) in gen8_ctx_workarounds_init() argument
235 wa_masked_en(wal, INSTPM, INSTPM_FORCE_ORDERING); in gen8_ctx_workarounds_init()
238 wa_masked_en(wal, MI_MODE, ASYNC_FLIP_PERF_DISABLE); in gen8_ctx_workarounds_init()
241 wa_masked_en(wal, GEN8_ROW_CHICKEN, in gen8_ctx_workarounds_init()
250 wa_masked_en(wal, HDC_CHICKEN0, in gen8_ctx_workarounds_init()
262 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE); in gen8_ctx_workarounds_init()
265 wa_masked_en(wal, CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE); in gen8_ctx_workarounds_init()
275 wa_masked_field_set(wal, GEN7_GT_MODE, in gen8_ctx_workarounds_init()
281 struct i915_wa_list *wal) in bdw_ctx_workarounds_init() argument
285 gen8_ctx_workarounds_init(engine, wal); in bdw_ctx_workarounds_init()
288 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); in bdw_ctx_workarounds_init()
295 wa_masked_en(wal, GEN7_ROW_CHICKEN2, in bdw_ctx_workarounds_init()
298 wa_masked_en(wal, HALF_SLICE_CHICKEN3, in bdw_ctx_workarounds_init()
301 wa_masked_en(wal, HDC_CHICKEN0, in bdw_ctx_workarounds_init()
309 struct i915_wa_list *wal) in chv_ctx_workarounds_init() argument
311 gen8_ctx_workarounds_init(engine, wal); in chv_ctx_workarounds_init()
314 wa_masked_en(wal, GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); in chv_ctx_workarounds_init()
317 wa_masked_en(wal, HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X); in chv_ctx_workarounds_init()
321 struct i915_wa_list *wal) in gen9_ctx_workarounds_init() argument
331 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in gen9_ctx_workarounds_init()
333 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, in gen9_ctx_workarounds_init()
339 wa_masked_en(wal, GEN8_ROW_CHICKEN, in gen9_ctx_workarounds_init()
345 wa_masked_en(wal, GEN9_HALF_SLICE_CHICKEN7, in gen9_ctx_workarounds_init()
351 wa_masked_en(wal, CACHE_MODE_1, in gen9_ctx_workarounds_init()
356 wa_masked_dis(wal, GEN9_HALF_SLICE_CHICKEN5, in gen9_ctx_workarounds_init()
360 wa_masked_en(wal, HDC_CHICKEN0, in gen9_ctx_workarounds_init()
378 wa_masked_en(wal, HDC_CHICKEN0, in gen9_ctx_workarounds_init()
386 wa_masked_en(wal, HALF_SLICE_CHICKEN3, in gen9_ctx_workarounds_init()
390 wa_masked_en(wal, HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE); in gen9_ctx_workarounds_init()
404 wa_masked_dis(wal, GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL); in gen9_ctx_workarounds_init()
407 wa_masked_field_set(wal, GEN8_CS_CHICKEN1, in gen9_ctx_workarounds_init()
413 wa_masked_en(wal, GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ); in gen9_ctx_workarounds_init()
417 struct i915_wa_list *wal) in skl_tune_iz_hashing() argument
447 wa_masked_field_set(wal, GEN7_GT_MODE, in skl_tune_iz_hashing()
457 struct i915_wa_list *wal) in skl_ctx_workarounds_init() argument
459 gen9_ctx_workarounds_init(engine, wal); in skl_ctx_workarounds_init()
460 skl_tune_iz_hashing(engine, wal); in skl_ctx_workarounds_init()
464 struct i915_wa_list *wal) in bxt_ctx_workarounds_init() argument
466 gen9_ctx_workarounds_init(engine, wal); in bxt_ctx_workarounds_init()
469 wa_masked_en(wal, GEN8_ROW_CHICKEN, in bxt_ctx_workarounds_init()
473 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in bxt_ctx_workarounds_init()
478 struct i915_wa_list *wal) in kbl_ctx_workarounds_init() argument
482 gen9_ctx_workarounds_init(engine, wal); in kbl_ctx_workarounds_init()
486 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in kbl_ctx_workarounds_init()
490 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1, in kbl_ctx_workarounds_init()
495 struct i915_wa_list *wal) in glk_ctx_workarounds_init() argument
497 gen9_ctx_workarounds_init(engine, wal); in glk_ctx_workarounds_init()
500 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in glk_ctx_workarounds_init()
505 struct i915_wa_list *wal) in cfl_ctx_workarounds_init() argument
507 gen9_ctx_workarounds_init(engine, wal); in cfl_ctx_workarounds_init()
510 wa_masked_en(wal, COMMON_SLICE_CHICKEN2, in cfl_ctx_workarounds_init()
514 wa_masked_en(wal, GEN7_HALF_SLICE_CHICKEN1, in cfl_ctx_workarounds_init()
519 struct i915_wa_list *wal) in icl_ctx_workarounds_init() argument
522 wa_write(wal, in icl_ctx_workarounds_init()
534 wa_masked_en(wal, ICL_HDC_MODE, HDC_FORCE_NON_COHERENT); in icl_ctx_workarounds_init()
537 wa_add(wal, GEN10_CACHE_MODE_SS, 0, in icl_ctx_workarounds_init()
543 wa_masked_field_set(wal, GEN8_CS_CHICKEN1, in icl_ctx_workarounds_init()
548 wa_masked_en(wal, GEN10_SAMPLER_MODE, in icl_ctx_workarounds_init()
552 wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID); in icl_ctx_workarounds_init()
553 wa_write_clr_set(wal, IVB_FBC_RT_BASE_UPPER, in icl_ctx_workarounds_init()
558 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU); in icl_ctx_workarounds_init()
566 struct i915_wa_list *wal) in gen12_ctx_gt_tuning_init() argument
581 wa_add(wal, in gen12_ctx_gt_tuning_init()
589 struct i915_wa_list *wal) in gen12_ctx_workarounds_init() argument
591 gen12_ctx_gt_tuning_init(engine, wal); in gen12_ctx_workarounds_init()
605 wa_masked_en(wal, GEN11_COMMON_SLICE_CHICKEN3, in gen12_ctx_workarounds_init()
609 wa_masked_field_set(wal, GEN8_CS_CHICKEN1, in gen12_ctx_workarounds_init()
619 wa_add(wal, in gen12_ctx_workarounds_init()
627 struct i915_wa_list *wal) in dg1_ctx_workarounds_init() argument
629 gen12_ctx_workarounds_init(engine, wal); in dg1_ctx_workarounds_init()
632 wa_masked_dis(wal, GEN11_COMMON_SLICE_CHICKEN3, in dg1_ctx_workarounds_init()
636 wa_masked_en(wal, HIZ_CHICKEN, in dg1_ctx_workarounds_init()
641 struct i915_wa_list *wal) in fakewa_disable_nestedbb_mode() argument
668 wa_masked_dis(wal, RING_MI_MODE(engine->mmio_base), TGL_NESTED_BB_EN); in fakewa_disable_nestedbb_mode()
672 struct i915_wa_list *wal) in gen12_ctx_gt_mocs_init() argument
683 wa_write_clr_set(wal, in gen12_ctx_gt_mocs_init()
698 struct i915_wa_list *wal) in gen12_ctx_gt_fake_wa_init() argument
701 fakewa_disable_nestedbb_mode(engine, wal); in gen12_ctx_gt_fake_wa_init()
703 gen12_ctx_gt_mocs_init(engine, wal); in gen12_ctx_gt_fake_wa_init()
708 struct i915_wa_list *wal, in __intel_engine_init_ctx_wa() argument
713 wa_init_start(wal, name, engine->name); in __intel_engine_init_ctx_wa()
721 gen12_ctx_gt_fake_wa_init(engine, wal); in __intel_engine_init_ctx_wa()
727 dg1_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
729 gen12_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
731 icl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
733 cfl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
735 glk_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
737 kbl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
739 bxt_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
741 skl_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
743 chv_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
745 bdw_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
747 gen7_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
749 gen6_ctx_workarounds_init(engine, wal); in __intel_engine_init_ctx_wa()
756 wa_init_finish(wal); in __intel_engine_init_ctx_wa()
766 struct i915_wa_list *wal = &rq->engine->ctx_wa_list; in intel_engine_emit_ctx_wa() local
772 if (wal->count == 0) in intel_engine_emit_ctx_wa()
779 cs = intel_ring_begin(rq, (wal->count * 2 + 2)); in intel_engine_emit_ctx_wa()
783 *cs++ = MI_LOAD_REGISTER_IMM(wal->count); in intel_engine_emit_ctx_wa()
784 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in intel_engine_emit_ctx_wa()
801 struct i915_wa_list *wal) in gen4_gt_workarounds_init() argument
804 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); in gen4_gt_workarounds_init()
808 g4x_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in g4x_gt_workarounds_init() argument
810 gen4_gt_workarounds_init(gt, wal); in g4x_gt_workarounds_init()
813 wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE); in g4x_gt_workarounds_init()
817 ilk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in ilk_gt_workarounds_init() argument
819 g4x_gt_workarounds_init(gt, wal); in ilk_gt_workarounds_init()
821 wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED); in ilk_gt_workarounds_init()
825 snb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in snb_gt_workarounds_init() argument
830 ivb_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in ivb_gt_workarounds_init() argument
833 wa_masked_dis(wal, in ivb_gt_workarounds_init()
838 wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL); in ivb_gt_workarounds_init()
839 wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE); in ivb_gt_workarounds_init()
842 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); in ivb_gt_workarounds_init()
846 vlv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in vlv_gt_workarounds_init() argument
849 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); in vlv_gt_workarounds_init()
855 wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); in vlv_gt_workarounds_init()
859 hsw_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in hsw_gt_workarounds_init() argument
862 wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); in hsw_gt_workarounds_init()
864 wa_add(wal, in hsw_gt_workarounds_init()
870 wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME); in hsw_gt_workarounds_init()
874 gen9_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in gen9_gt_workarounds_init() argument
880 wa_write_or(wal, in gen9_gt_workarounds_init()
890 wa_write_or(wal, in gen9_gt_workarounds_init()
896 wa_write_or(wal, in gen9_gt_workarounds_init()
902 skl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in skl_gt_workarounds_init() argument
904 gen9_gt_workarounds_init(gt, wal); in skl_gt_workarounds_init()
907 wa_write_or(wal, in skl_gt_workarounds_init()
913 wa_write_or(wal, in skl_gt_workarounds_init()
919 kbl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in kbl_gt_workarounds_init() argument
921 gen9_gt_workarounds_init(gt, wal); in kbl_gt_workarounds_init()
925 wa_write_or(wal, in kbl_gt_workarounds_init()
930 wa_write_or(wal, in kbl_gt_workarounds_init()
935 wa_write_or(wal, in kbl_gt_workarounds_init()
941 glk_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in glk_gt_workarounds_init() argument
943 gen9_gt_workarounds_init(gt, wal); in glk_gt_workarounds_init()
947 cfl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in cfl_gt_workarounds_init() argument
949 gen9_gt_workarounds_init(gt, wal); in cfl_gt_workarounds_init()
952 wa_write_or(wal, in cfl_gt_workarounds_init()
957 wa_write_or(wal, in cfl_gt_workarounds_init()
962 static void __set_mcr_steering(struct i915_wa_list *wal, in __set_mcr_steering() argument
971 wa_write_clr_set(wal, steering_reg, mcr_mask, mcr); in __set_mcr_steering()
974 static void __add_mcr_wa(struct intel_gt *gt, struct i915_wa_list *wal, in __add_mcr_wa() argument
979 __set_mcr_steering(wal, GEN8_MCR_SELECTOR, slice, subslice); in __add_mcr_wa()
983 icl_wa_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) in icl_wa_init_mcr() argument
1011 __add_mcr_wa(gt, wal, slice, subslice); in icl_wa_init_mcr()
1015 xehp_init_mcr(struct intel_gt *gt, struct i915_wa_list *wal) in xehp_init_mcr() argument
1080 __add_mcr_wa(gt, wal, slice, subslice); in xehp_init_mcr()
1091 __set_mcr_steering(wal, MCFG_MCR_SELECTOR, 0, 2); in xehp_init_mcr()
1092 __set_mcr_steering(wal, SF_MCR_SELECTOR, 0, 2); in xehp_init_mcr()
1096 icl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in icl_gt_workarounds_init() argument
1100 icl_wa_init_mcr(gt, wal); in icl_gt_workarounds_init()
1103 wa_write_clr_set(wal, in icl_gt_workarounds_init()
1111 wa_write_or(wal, in icl_gt_workarounds_init()
1119 wa_write_or(wal, in icl_gt_workarounds_init()
1126 wa_write_or(wal, in icl_gt_workarounds_init()
1131 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, in icl_gt_workarounds_init()
1135 wa_write_or(wal, in icl_gt_workarounds_init()
1142 wa_write_or(wal, in icl_gt_workarounds_init()
1150 wa_write_clr(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE); in icl_gt_workarounds_init()
1160 wa_14011060649(struct intel_gt *gt, struct i915_wa_list *wal) in wa_14011060649() argument
1170 wa_write_or(wal, VDBOX_CGCTL3F10(engine->mmio_base), in wa_14011060649()
1176 gen12_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in gen12_gt_workarounds_init() argument
1178 icl_wa_init_mcr(gt, wal); in gen12_gt_workarounds_init()
1181 wa_14011060649(gt, wal); in gen12_gt_workarounds_init()
1184 wa_write_or(wal, GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE); in gen12_gt_workarounds_init()
1188 tgl_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in tgl_gt_workarounds_init() argument
1192 gen12_gt_workarounds_init(gt, wal); in tgl_gt_workarounds_init()
1196 wa_write_or(wal, in tgl_gt_workarounds_init()
1202 wa_write_or(wal, in tgl_gt_workarounds_init()
1208 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, in tgl_gt_workarounds_init()
1213 dg1_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in dg1_gt_workarounds_init() argument
1217 gen12_gt_workarounds_init(gt, wal); in dg1_gt_workarounds_init()
1221 wa_write_or(wal, in dg1_gt_workarounds_init()
1227 wa_write_or(wal, in dg1_gt_workarounds_init()
1234 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, in dg1_gt_workarounds_init()
1239 xehpsdv_gt_workarounds_init(struct intel_gt *gt, struct i915_wa_list *wal) in xehpsdv_gt_workarounds_init() argument
1241 xehp_init_mcr(gt, wal); in xehpsdv_gt_workarounds_init()
1245 gt_init_workarounds(struct intel_gt *gt, struct i915_wa_list *wal) in gt_init_workarounds() argument
1250 xehpsdv_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1252 dg1_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1254 tgl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1256 gen12_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1258 icl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1260 cfl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1262 glk_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1264 kbl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1266 gen9_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1268 skl_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1270 hsw_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1272 vlv_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1274 ivb_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1276 snb_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1278 ilk_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1280 g4x_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1282 gen4_gt_workarounds_init(gt, wal); in gt_init_workarounds()
1291 struct i915_wa_list *wal = >->wa_list; in intel_gt_init_workarounds() local
1293 wa_init_start(wal, "GT", "global"); in intel_gt_init_workarounds()
1294 gt_init_workarounds(gt, wal); in intel_gt_init_workarounds()
1295 wa_init_finish(wal); in intel_gt_init_workarounds()
1299 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal) in wal_get_fw_for_rmw() argument
1305 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in wal_get_fw_for_rmw()
1329 wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal) in wa_list_apply() argument
1337 if (!wal->count) in wa_list_apply()
1340 fw = wal_get_fw_for_rmw(uncore, wal); in wa_list_apply()
1345 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_apply()
1356 wal->name, "application"); in wa_list_apply()
1369 const struct i915_wa_list *wal, in wa_list_verify() argument
1379 fw = wal_get_fw_for_rmw(uncore, wal); in wa_list_verify()
1384 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in wa_list_verify()
1387 wal->name, from); in wa_list_verify()
1416 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags) in whitelist_reg_ext() argument
1422 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)) in whitelist_reg_ext()
1429 _wa_add(wal, &wa); in whitelist_reg_ext()
1433 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg) in whitelist_reg() argument
1435 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW); in whitelist_reg()
1670 const struct i915_wa_list *wal = &engine->whitelist; in intel_engine_apply_whitelist() local
1676 if (!wal->count) in intel_engine_apply_whitelist()
1679 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) in intel_engine_apply_whitelist()
1699 engine_fake_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in engine_fake_wa_init() argument
1710 wa_masked_field_set(wal, in engine_fake_wa_init()
1717 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in rcs_engine_wa_init() argument
1727 wa_write_or(wal, in rcs_engine_wa_init()
1737 wa_write_or(wal, in rcs_engine_wa_init()
1745 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ); in rcs_engine_wa_init()
1753 wa_write_or(wal, GEN7_FF_THREAD_MODE, in rcs_engine_wa_init()
1761 wa_masked_en(wal, in rcs_engine_wa_init()
1770 wa_masked_en(wal, GEN7_ROW_CHICKEN2, in rcs_engine_wa_init()
1777 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH); in rcs_engine_wa_init()
1793 wa_masked_en(wal, in rcs_engine_wa_init()
1802 wa_masked_en(wal, in rcs_engine_wa_init()
1809 wa_masked_en(wal, in rcs_engine_wa_init()
1817 wa_write_or(wal, in rcs_engine_wa_init()
1825 wa_write_clr_set(wal, in rcs_engine_wa_init()
1829 wa_write_clr_set(wal, in rcs_engine_wa_init()
1838 wa_write_or(wal, in rcs_engine_wa_init()
1843 wa_write_or(wal, in rcs_engine_wa_init()
1848 wa_write_clr_set(wal, in rcs_engine_wa_init()
1854 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS, in rcs_engine_wa_init()
1861 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, in rcs_engine_wa_init()
1868 wa_write_or(wal, in rcs_engine_wa_init()
1873 wa_masked_en(wal, in rcs_engine_wa_init()
1880 wa_masked_en(wal, in rcs_engine_wa_init()
1890 wa_write_or(wal, in rcs_engine_wa_init()
1897 wa_masked_en(wal, in rcs_engine_wa_init()
1904 wa_masked_en(wal, in rcs_engine_wa_init()
1909 wa_write_or(wal, in rcs_engine_wa_init()
1915 wa_write_clr_set(wal, in rcs_engine_wa_init()
1922 wa_write_or(wal, in rcs_engine_wa_init()
1927 wa_write_clr_set(wal, GEN9_SCRATCH_LNCF1, in rcs_engine_wa_init()
1929 wa_write_clr_set(wal, GEN8_L3SQCREG4, in rcs_engine_wa_init()
1931 wa_write_clr_set(wal, GEN9_SCRATCH1, in rcs_engine_wa_init()
1937 wa_masked_en(wal, in rcs_engine_wa_init()
1940 wa_masked_dis(wal, in rcs_engine_wa_init()
1948 wa_masked_en(wal, in rcs_engine_wa_init()
1958 wa_write_clr_set(wal, in rcs_engine_wa_init()
1967 wa_masked_en(wal, in rcs_engine_wa_init()
1975 wa_masked_en(wal, in rcs_engine_wa_init()
1981 wa_masked_dis(wal, in rcs_engine_wa_init()
1992 wa_write_clr_set(wal, in rcs_engine_wa_init()
2001 wa_masked_en(wal, in rcs_engine_wa_init()
2008 wa_masked_en(wal, in rcs_engine_wa_init()
2013 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE); in rcs_engine_wa_init()
2020 wa_masked_en(wal, in rcs_engine_wa_init()
2032 wa_masked_field_set(wal, in rcs_engine_wa_init()
2046 wa_masked_en(wal, in rcs_engine_wa_init()
2056 wa_masked_en(wal, in rcs_engine_wa_init()
2061 wa_masked_en(wal, in rcs_engine_wa_init()
2065 wa_masked_en(wal, in rcs_engine_wa_init()
2085 wa_masked_field_set(wal, in rcs_engine_wa_init()
2091 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); in rcs_engine_wa_init()
2099 wa_masked_dis(wal, in rcs_engine_wa_init()
2106 wa_add(wal, MI_MODE, in rcs_engine_wa_init()
2122 wa_add(wal, ECOSKPD, in rcs_engine_wa_init()
2129 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) in xcs_engine_wa_init() argument
2135 wa_write(wal, in xcs_engine_wa_init()
2142 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) in engine_init_workarounds() argument
2147 engine_fake_wa_init(engine, wal); in engine_init_workarounds()
2150 rcs_engine_wa_init(engine, wal); in engine_init_workarounds()
2152 xcs_engine_wa_init(engine, wal); in engine_init_workarounds()
2157 struct i915_wa_list *wal = &engine->wa_list; in intel_engine_init_workarounds() local
2162 wa_init_start(wal, "engine", engine->name); in intel_engine_init_workarounds()
2163 engine_init_workarounds(engine, wal); in intel_engine_init_workarounds()
2164 wa_init_finish(wal); in intel_engine_init_workarounds()
2236 const struct i915_wa_list *wal, in wa_list_srm() argument
2248 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_srm()
2257 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in wa_list_srm()
2274 const struct i915_wa_list * const wal, in engine_wa_list_verify() argument
2285 if (!wal->count) in engine_wa_list_verify()
2289 wal->count * sizeof(u32)); in engine_wa_list_verify()
2317 err = wa_list_srm(rq, wal, vma); in engine_wa_list_verify()
2339 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { in engine_wa_list_verify()
2343 if (!wa_verify(wa, results[i], wal->name, from)) in engine_wa_list_verify()