1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include <linux/sched/clock.h>
7
8 #include "i915_drv.h"
9 #include "i915_irq.h"
10 #include "intel_breadcrumbs.h"
11 #include "intel_gt.h"
12 #include "intel_gt_irq.h"
13 #include "intel_lrc_reg.h"
14 #include "intel_uncore.h"
15 #include "intel_rps.h"
16 #include "pxp/intel_pxp_irq.h"
17
guc_irq_handler(struct intel_guc * guc,u16 iir)18 static void guc_irq_handler(struct intel_guc *guc, u16 iir)
19 {
20 if (iir & GUC_INTR_GUC2HOST)
21 intel_guc_to_host_event_handler(guc);
22 }
23
24 static u32
gen11_gt_engine_identity(struct intel_gt * gt,const unsigned int bank,const unsigned int bit)25 gen11_gt_engine_identity(struct intel_gt *gt,
26 const unsigned int bank, const unsigned int bit)
27 {
28 void __iomem * const regs = gt->uncore->regs;
29 u32 timeout_ts;
30 u32 ident;
31
32 lockdep_assert_held(>->irq_lock);
33
34 raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
35
36 /*
37 * NB: Specs do not specify how long to spin wait,
38 * so we do ~100us as an educated guess.
39 */
40 timeout_ts = (local_clock() >> 10) + 100;
41 do {
42 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
43 } while (!(ident & GEN11_INTR_DATA_VALID) &&
44 !time_after32(local_clock() >> 10, timeout_ts));
45
46 if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
47 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
48 bank, bit, ident);
49 return 0;
50 }
51
52 raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
53 GEN11_INTR_DATA_VALID);
54
55 return ident;
56 }
57
58 static void
gen11_other_irq_handler(struct intel_gt * gt,const u8 instance,const u16 iir)59 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
60 const u16 iir)
61 {
62 if (instance == OTHER_GUC_INSTANCE)
63 return guc_irq_handler(>->uc.guc, iir);
64
65 if (instance == OTHER_GTPM_INSTANCE)
66 return gen11_rps_irq_handler(>->rps, iir);
67
68 if (instance == OTHER_KCR_INSTANCE)
69 return intel_pxp_irq_handler(>->pxp, iir);
70
71 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
72 instance, iir);
73 }
74
75 static void
gen11_engine_irq_handler(struct intel_gt * gt,const u8 class,const u8 instance,const u16 iir)76 gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
77 const u8 instance, const u16 iir)
78 {
79 struct intel_engine_cs *engine;
80
81 if (instance <= MAX_ENGINE_INSTANCE)
82 engine = gt->engine_class[class][instance];
83 else
84 engine = NULL;
85
86 if (likely(engine))
87 return intel_engine_cs_irq(engine, iir);
88
89 WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
90 class, instance);
91 }
92
93 static void
gen11_gt_identity_handler(struct intel_gt * gt,const u32 identity)94 gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
95 {
96 const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
97 const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
98 const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
99
100 if (unlikely(!intr))
101 return;
102
103 if (class <= COPY_ENGINE_CLASS)
104 return gen11_engine_irq_handler(gt, class, instance, intr);
105
106 if (class == OTHER_CLASS)
107 return gen11_other_irq_handler(gt, instance, intr);
108
109 WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
110 class, instance, intr);
111 }
112
113 static void
gen11_gt_bank_handler(struct intel_gt * gt,const unsigned int bank)114 gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
115 {
116 void __iomem * const regs = gt->uncore->regs;
117 unsigned long intr_dw;
118 unsigned int bit;
119
120 lockdep_assert_held(>->irq_lock);
121
122 intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
123
124 for_each_set_bit(bit, &intr_dw, 32) {
125 const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
126
127 gen11_gt_identity_handler(gt, ident);
128 }
129
130 /* Clear must be after shared has been served for engine */
131 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
132 }
133
gen11_gt_irq_handler(struct intel_gt * gt,const u32 master_ctl)134 void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
135 {
136 unsigned int bank;
137
138 spin_lock(>->irq_lock);
139
140 for (bank = 0; bank < 2; bank++) {
141 if (master_ctl & GEN11_GT_DW_IRQ(bank))
142 gen11_gt_bank_handler(gt, bank);
143 }
144
145 spin_unlock(>->irq_lock);
146 }
147
gen11_gt_reset_one_iir(struct intel_gt * gt,const unsigned int bank,const unsigned int bit)148 bool gen11_gt_reset_one_iir(struct intel_gt *gt,
149 const unsigned int bank, const unsigned int bit)
150 {
151 void __iomem * const regs = gt->uncore->regs;
152 u32 dw;
153
154 lockdep_assert_held(>->irq_lock);
155
156 dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
157 if (dw & BIT(bit)) {
158 /*
159 * According to the BSpec, DW_IIR bits cannot be cleared without
160 * first servicing the Selector & Shared IIR registers.
161 */
162 gen11_gt_engine_identity(gt, bank, bit);
163
164 /*
165 * We locked GT INT DW by reading it. If we want to (try
166 * to) recover from this successfully, we need to clear
167 * our bit, otherwise we are locking the register for
168 * everybody.
169 */
170 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
171
172 return true;
173 }
174
175 return false;
176 }
177
gen11_gt_irq_reset(struct intel_gt * gt)178 void gen11_gt_irq_reset(struct intel_gt *gt)
179 {
180 struct intel_uncore *uncore = gt->uncore;
181
182 /* Disable RCS, BCS, VCS and VECS class engines. */
183 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
184 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0);
185
186 /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */
187 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~0);
188 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~0);
189 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~0);
190 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~0);
191 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
192 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~0);
193 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
194 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~0);
195 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0);
196 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
197 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~0);
198
199 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
200 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
201 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
202 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0);
203
204 intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_ENABLE, 0);
205 intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_MASK, ~0);
206 }
207
gen11_gt_irq_postinstall(struct intel_gt * gt)208 void gen11_gt_irq_postinstall(struct intel_gt *gt)
209 {
210 struct intel_uncore *uncore = gt->uncore;
211 u32 irqs = GT_RENDER_USER_INTERRUPT;
212 u32 dmask;
213 u32 smask;
214
215 if (!intel_uc_wants_guc_submission(>->uc))
216 irqs |= GT_CS_MASTER_ERROR_INTERRUPT |
217 GT_CONTEXT_SWITCH_INTERRUPT |
218 GT_WAIT_SEMAPHORE_INTERRUPT;
219
220 dmask = irqs << 16 | irqs;
221 smask = irqs << 16;
222
223 BUILD_BUG_ON(irqs & 0xffff0000);
224
225 /* Enable RCS, BCS, VCS and VECS class interrupts. */
226 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
227 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
228
229 /* Unmask irqs on RCS, BCS, VCS and VECS engines. */
230 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
231 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
232 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
233 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
234 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
235 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~dmask);
236 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
237 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~dmask);
238 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
239 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
240 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask);
241 /*
242 * RPS interrupts will get enabled/disabled on demand when RPS itself
243 * is enabled/disabled.
244 */
245 gt->pm_ier = 0x0;
246 gt->pm_imr = ~gt->pm_ier;
247 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
248 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
249
250 /* Same thing for GuC interrupts */
251 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
252 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0);
253 }
254
gen5_gt_irq_handler(struct intel_gt * gt,u32 gt_iir)255 void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
256 {
257 if (gt_iir & GT_RENDER_USER_INTERRUPT)
258 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
259 gt_iir);
260
261 if (gt_iir & ILK_BSD_USER_INTERRUPT)
262 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
263 gt_iir);
264 }
265
gen7_parity_error_irq_handler(struct intel_gt * gt,u32 iir)266 static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
267 {
268 if (!HAS_L3_DPF(gt->i915))
269 return;
270
271 spin_lock(>->irq_lock);
272 gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
273 spin_unlock(>->irq_lock);
274
275 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
276 gt->i915->l3_parity.which_slice |= 1 << 1;
277
278 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
279 gt->i915->l3_parity.which_slice |= 1 << 0;
280
281 schedule_work(>->i915->l3_parity.error_work);
282 }
283
gen6_gt_irq_handler(struct intel_gt * gt,u32 gt_iir)284 void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
285 {
286 if (gt_iir & GT_RENDER_USER_INTERRUPT)
287 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
288 gt_iir);
289
290 if (gt_iir & GT_BSD_USER_INTERRUPT)
291 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
292 gt_iir >> 12);
293
294 if (gt_iir & GT_BLT_USER_INTERRUPT)
295 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
296 gt_iir >> 22);
297
298 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
299 GT_BSD_CS_ERROR_INTERRUPT |
300 GT_CS_MASTER_ERROR_INTERRUPT))
301 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
302
303 if (gt_iir & GT_PARITY_ERROR(gt->i915))
304 gen7_parity_error_irq_handler(gt, gt_iir);
305 }
306
gen8_gt_irq_handler(struct intel_gt * gt,u32 master_ctl)307 void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl)
308 {
309 void __iomem * const regs = gt->uncore->regs;
310 u32 iir;
311
312 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
313 iir = raw_reg_read(regs, GEN8_GT_IIR(0));
314 if (likely(iir)) {
315 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
316 iir >> GEN8_RCS_IRQ_SHIFT);
317 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
318 iir >> GEN8_BCS_IRQ_SHIFT);
319 raw_reg_write(regs, GEN8_GT_IIR(0), iir);
320 }
321 }
322
323 if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
324 iir = raw_reg_read(regs, GEN8_GT_IIR(1));
325 if (likely(iir)) {
326 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
327 iir >> GEN8_VCS0_IRQ_SHIFT);
328 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1],
329 iir >> GEN8_VCS1_IRQ_SHIFT);
330 raw_reg_write(regs, GEN8_GT_IIR(1), iir);
331 }
332 }
333
334 if (master_ctl & GEN8_GT_VECS_IRQ) {
335 iir = raw_reg_read(regs, GEN8_GT_IIR(3));
336 if (likely(iir)) {
337 intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
338 iir >> GEN8_VECS_IRQ_SHIFT);
339 raw_reg_write(regs, GEN8_GT_IIR(3), iir);
340 }
341 }
342
343 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
344 iir = raw_reg_read(regs, GEN8_GT_IIR(2));
345 if (likely(iir)) {
346 gen6_rps_irq_handler(>->rps, iir);
347 guc_irq_handler(>->uc.guc, iir >> 16);
348 raw_reg_write(regs, GEN8_GT_IIR(2), iir);
349 }
350 }
351 }
352
gen8_gt_irq_reset(struct intel_gt * gt)353 void gen8_gt_irq_reset(struct intel_gt *gt)
354 {
355 struct intel_uncore *uncore = gt->uncore;
356
357 GEN8_IRQ_RESET_NDX(uncore, GT, 0);
358 GEN8_IRQ_RESET_NDX(uncore, GT, 1);
359 GEN8_IRQ_RESET_NDX(uncore, GT, 2);
360 GEN8_IRQ_RESET_NDX(uncore, GT, 3);
361 }
362
gen8_gt_irq_postinstall(struct intel_gt * gt)363 void gen8_gt_irq_postinstall(struct intel_gt *gt)
364 {
365 /* These are interrupts we'll toggle with the ring mask register */
366 const u32 irqs =
367 GT_CS_MASTER_ERROR_INTERRUPT |
368 GT_RENDER_USER_INTERRUPT |
369 GT_CONTEXT_SWITCH_INTERRUPT |
370 GT_WAIT_SEMAPHORE_INTERRUPT;
371 const u32 gt_interrupts[] = {
372 irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT,
373 irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT,
374 0,
375 irqs << GEN8_VECS_IRQ_SHIFT,
376 };
377 struct intel_uncore *uncore = gt->uncore;
378
379 gt->pm_ier = 0x0;
380 gt->pm_imr = ~gt->pm_ier;
381 GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
382 GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
383 /*
384 * RPS interrupts will get enabled/disabled on demand when RPS itself
385 * is enabled/disabled. Same wil be the case for GuC interrupts.
386 */
387 GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
388 GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
389 }
390
gen5_gt_update_irq(struct intel_gt * gt,u32 interrupt_mask,u32 enabled_irq_mask)391 static void gen5_gt_update_irq(struct intel_gt *gt,
392 u32 interrupt_mask,
393 u32 enabled_irq_mask)
394 {
395 lockdep_assert_held(>->irq_lock);
396
397 GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
398
399 gt->gt_imr &= ~interrupt_mask;
400 gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
401 intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
402 }
403
gen5_gt_enable_irq(struct intel_gt * gt,u32 mask)404 void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
405 {
406 gen5_gt_update_irq(gt, mask, mask);
407 intel_uncore_posting_read_fw(gt->uncore, GTIMR);
408 }
409
gen5_gt_disable_irq(struct intel_gt * gt,u32 mask)410 void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
411 {
412 gen5_gt_update_irq(gt, mask, 0);
413 }
414
gen5_gt_irq_reset(struct intel_gt * gt)415 void gen5_gt_irq_reset(struct intel_gt *gt)
416 {
417 struct intel_uncore *uncore = gt->uncore;
418
419 GEN3_IRQ_RESET(uncore, GT);
420 if (GRAPHICS_VER(gt->i915) >= 6)
421 GEN3_IRQ_RESET(uncore, GEN6_PM);
422 }
423
gen5_gt_irq_postinstall(struct intel_gt * gt)424 void gen5_gt_irq_postinstall(struct intel_gt *gt)
425 {
426 struct intel_uncore *uncore = gt->uncore;
427 u32 pm_irqs = 0;
428 u32 gt_irqs = 0;
429
430 gt->gt_imr = ~0;
431 if (HAS_L3_DPF(gt->i915)) {
432 /* L3 parity interrupt is always unmasked. */
433 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
434 gt_irqs |= GT_PARITY_ERROR(gt->i915);
435 }
436
437 gt_irqs |= GT_RENDER_USER_INTERRUPT;
438 if (GRAPHICS_VER(gt->i915) == 5)
439 gt_irqs |= ILK_BSD_USER_INTERRUPT;
440 else
441 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
442
443 GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
444
445 if (GRAPHICS_VER(gt->i915) >= 6) {
446 /*
447 * RPS interrupts will get enabled/disabled on demand when RPS
448 * itself is enabled/disabled.
449 */
450 if (HAS_ENGINE(gt, VECS0)) {
451 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
452 gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
453 }
454
455 gt->pm_imr = 0xffffffff;
456 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);
457 }
458 }
459