0001
0002
0003
0004
0005
0006 #include <linux/sched/clock.h>
0007
0008 #include "i915_drv.h"
0009 #include "i915_irq.h"
0010 #include "intel_breadcrumbs.h"
0011 #include "intel_gt.h"
0012 #include "intel_gt_irq.h"
0013 #include "intel_gt_regs.h"
0014 #include "intel_uncore.h"
0015 #include "intel_rps.h"
0016 #include "pxp/intel_pxp_irq.h"
0017
0018 static void guc_irq_handler(struct intel_guc *guc, u16 iir)
0019 {
0020 if (iir & GUC_INTR_GUC2HOST)
0021 intel_guc_to_host_event_handler(guc);
0022 }
0023
0024 static u32
0025 gen11_gt_engine_identity(struct intel_gt *gt,
0026 const unsigned int bank, const unsigned int bit)
0027 {
0028 void __iomem * const regs = gt->uncore->regs;
0029 u32 timeout_ts;
0030 u32 ident;
0031
0032 lockdep_assert_held(>->irq_lock);
0033
0034 raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
0035
0036
0037
0038
0039
0040 timeout_ts = (local_clock() >> 10) + 100;
0041 do {
0042 ident = raw_reg_read(regs, GEN11_INTR_IDENTITY_REG(bank));
0043 } while (!(ident & GEN11_INTR_DATA_VALID) &&
0044 !time_after32(local_clock() >> 10, timeout_ts));
0045
0046 if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) {
0047 DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n",
0048 bank, bit, ident);
0049 return 0;
0050 }
0051
0052 raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
0053 GEN11_INTR_DATA_VALID);
0054
0055 return ident;
0056 }
0057
0058 static void
0059 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
0060 const u16 iir)
0061 {
0062 if (instance == OTHER_GUC_INSTANCE)
0063 return guc_irq_handler(>->uc.guc, iir);
0064
0065 if (instance == OTHER_GTPM_INSTANCE)
0066 return gen11_rps_irq_handler(>->rps, iir);
0067
0068 if (instance == OTHER_KCR_INSTANCE)
0069 return intel_pxp_irq_handler(>->pxp, iir);
0070
0071 if (instance == OTHER_GSC_INSTANCE)
0072 return intel_gsc_irq_handler(gt, iir);
0073
0074 WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
0075 instance, iir);
0076 }
0077
0078 static void
0079 gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
0080 const u8 instance, const u16 iir)
0081 {
0082 struct intel_engine_cs *engine;
0083
0084 if (instance <= MAX_ENGINE_INSTANCE)
0085 engine = gt->engine_class[class][instance];
0086 else
0087 engine = NULL;
0088
0089 if (likely(engine))
0090 return intel_engine_cs_irq(engine, iir);
0091
0092 WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
0093 class, instance);
0094 }
0095
0096 static void
0097 gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity)
0098 {
0099 const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
0100 const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
0101 const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
0102
0103 if (unlikely(!intr))
0104 return;
0105
0106 if (class <= COPY_ENGINE_CLASS || class == COMPUTE_CLASS)
0107 return gen11_engine_irq_handler(gt, class, instance, intr);
0108
0109 if (class == OTHER_CLASS)
0110 return gen11_other_irq_handler(gt, instance, intr);
0111
0112 WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
0113 class, instance, intr);
0114 }
0115
0116 static void
0117 gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
0118 {
0119 void __iomem * const regs = gt->uncore->regs;
0120 unsigned long intr_dw;
0121 unsigned int bit;
0122
0123 lockdep_assert_held(>->irq_lock);
0124
0125 intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
0126
0127 for_each_set_bit(bit, &intr_dw, 32) {
0128 const u32 ident = gen11_gt_engine_identity(gt, bank, bit);
0129
0130 gen11_gt_identity_handler(gt, ident);
0131 }
0132
0133
0134 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
0135 }
0136
0137 void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
0138 {
0139 unsigned int bank;
0140
0141 spin_lock(>->irq_lock);
0142
0143 for (bank = 0; bank < 2; bank++) {
0144 if (master_ctl & GEN11_GT_DW_IRQ(bank))
0145 gen11_gt_bank_handler(gt, bank);
0146 }
0147
0148 spin_unlock(>->irq_lock);
0149 }
0150
0151 bool gen11_gt_reset_one_iir(struct intel_gt *gt,
0152 const unsigned int bank, const unsigned int bit)
0153 {
0154 void __iomem * const regs = gt->uncore->regs;
0155 u32 dw;
0156
0157 lockdep_assert_held(>->irq_lock);
0158
0159 dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
0160 if (dw & BIT(bit)) {
0161
0162
0163
0164
0165 gen11_gt_engine_identity(gt, bank, bit);
0166
0167
0168
0169
0170
0171
0172
0173 raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
0174
0175 return true;
0176 }
0177
0178 return false;
0179 }
0180
0181 void gen11_gt_irq_reset(struct intel_gt *gt)
0182 {
0183 struct intel_uncore *uncore = gt->uncore;
0184
0185
0186 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, 0);
0187 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0);
0188 if (CCS_MASK(gt))
0189 intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, 0);
0190 if (HAS_HECI_GSC(gt->i915))
0191 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE, 0);
0192
0193
0194 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~0);
0195 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~0);
0196 if (HAS_ENGINE(gt, BCS1) || HAS_ENGINE(gt, BCS2))
0197 intel_uncore_write(uncore, XEHPC_BCS1_BCS2_INTR_MASK, ~0);
0198 if (HAS_ENGINE(gt, BCS3) || HAS_ENGINE(gt, BCS4))
0199 intel_uncore_write(uncore, XEHPC_BCS3_BCS4_INTR_MASK, ~0);
0200 if (HAS_ENGINE(gt, BCS5) || HAS_ENGINE(gt, BCS6))
0201 intel_uncore_write(uncore, XEHPC_BCS5_BCS6_INTR_MASK, ~0);
0202 if (HAS_ENGINE(gt, BCS7) || HAS_ENGINE(gt, BCS8))
0203 intel_uncore_write(uncore, XEHPC_BCS7_BCS8_INTR_MASK, ~0);
0204 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~0);
0205 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~0);
0206 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
0207 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~0);
0208 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
0209 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~0);
0210 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~0);
0211 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
0212 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~0);
0213 if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1))
0214 intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~0);
0215 if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3))
0216 intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~0);
0217 if (HAS_HECI_GSC(gt->i915))
0218 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_MASK, ~0);
0219
0220 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
0221 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
0222 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
0223 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0);
0224
0225 intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_ENABLE, 0);
0226 intel_uncore_write(uncore, GEN11_CRYPTO_RSVD_INTR_MASK, ~0);
0227 }
0228
0229 void gen11_gt_irq_postinstall(struct intel_gt *gt)
0230 {
0231 struct intel_uncore *uncore = gt->uncore;
0232 u32 irqs = GT_RENDER_USER_INTERRUPT;
0233 const u32 gsc_mask = GSC_IRQ_INTF(0) | GSC_IRQ_INTF(1);
0234 u32 dmask;
0235 u32 smask;
0236
0237 if (!intel_uc_wants_guc_submission(>->uc))
0238 irqs |= GT_CS_MASTER_ERROR_INTERRUPT |
0239 GT_CONTEXT_SWITCH_INTERRUPT |
0240 GT_WAIT_SEMAPHORE_INTERRUPT;
0241
0242 dmask = irqs << 16 | irqs;
0243 smask = irqs << 16;
0244
0245 BUILD_BUG_ON(irqs & 0xffff0000);
0246
0247
0248 intel_uncore_write(uncore, GEN11_RENDER_COPY_INTR_ENABLE, dmask);
0249 intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask);
0250 if (CCS_MASK(gt))
0251 intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, smask);
0252 if (HAS_HECI_GSC(gt->i915))
0253 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE,
0254 gsc_mask);
0255
0256
0257 intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask);
0258 intel_uncore_write(uncore, GEN11_BCS_RSVD_INTR_MASK, ~smask);
0259 if (HAS_ENGINE(gt, BCS1) || HAS_ENGINE(gt, BCS2))
0260 intel_uncore_write(uncore, XEHPC_BCS1_BCS2_INTR_MASK, ~dmask);
0261 if (HAS_ENGINE(gt, BCS3) || HAS_ENGINE(gt, BCS4))
0262 intel_uncore_write(uncore, XEHPC_BCS3_BCS4_INTR_MASK, ~dmask);
0263 if (HAS_ENGINE(gt, BCS5) || HAS_ENGINE(gt, BCS6))
0264 intel_uncore_write(uncore, XEHPC_BCS5_BCS6_INTR_MASK, ~dmask);
0265 if (HAS_ENGINE(gt, BCS7) || HAS_ENGINE(gt, BCS8))
0266 intel_uncore_write(uncore, XEHPC_BCS7_BCS8_INTR_MASK, ~dmask);
0267 intel_uncore_write(uncore, GEN11_VCS0_VCS1_INTR_MASK, ~dmask);
0268 intel_uncore_write(uncore, GEN11_VCS2_VCS3_INTR_MASK, ~dmask);
0269 if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5))
0270 intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~dmask);
0271 if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7))
0272 intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~dmask);
0273 intel_uncore_write(uncore, GEN11_VECS0_VECS1_INTR_MASK, ~dmask);
0274 if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3))
0275 intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask);
0276 if (HAS_ENGINE(gt, CCS0) || HAS_ENGINE(gt, CCS1))
0277 intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~dmask);
0278 if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3))
0279 intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~dmask);
0280 if (HAS_HECI_GSC(gt->i915))
0281 intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_MASK, ~gsc_mask);
0282
0283
0284
0285
0286
0287 gt->pm_ier = 0x0;
0288 gt->pm_imr = ~gt->pm_ier;
0289 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
0290 intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK, ~0);
0291
0292
0293 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
0294 intel_uncore_write(uncore, GEN11_GUC_SG_INTR_MASK, ~0);
0295 }
0296
0297 void gen5_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
0298 {
0299 if (gt_iir & GT_RENDER_USER_INTERRUPT)
0300 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
0301 gt_iir);
0302
0303 if (gt_iir & ILK_BSD_USER_INTERRUPT)
0304 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
0305 gt_iir);
0306 }
0307
0308 static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
0309 {
0310 if (!HAS_L3_DPF(gt->i915))
0311 return;
0312
0313 spin_lock(>->irq_lock);
0314 gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
0315 spin_unlock(>->irq_lock);
0316
0317 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
0318 gt->i915->l3_parity.which_slice |= 1 << 1;
0319
0320 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
0321 gt->i915->l3_parity.which_slice |= 1 << 0;
0322
0323 schedule_work(>->i915->l3_parity.error_work);
0324 }
0325
0326 void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir)
0327 {
0328 if (gt_iir & GT_RENDER_USER_INTERRUPT)
0329 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
0330 gt_iir);
0331
0332 if (gt_iir & GT_BSD_USER_INTERRUPT)
0333 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
0334 gt_iir >> 12);
0335
0336 if (gt_iir & GT_BLT_USER_INTERRUPT)
0337 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
0338 gt_iir >> 22);
0339
0340 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
0341 GT_BSD_CS_ERROR_INTERRUPT |
0342 GT_CS_MASTER_ERROR_INTERRUPT))
0343 DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
0344
0345 if (gt_iir & GT_PARITY_ERROR(gt->i915))
0346 gen7_parity_error_irq_handler(gt, gt_iir);
0347 }
0348
0349 void gen8_gt_irq_handler(struct intel_gt *gt, u32 master_ctl)
0350 {
0351 void __iomem * const regs = gt->uncore->regs;
0352 u32 iir;
0353
0354 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
0355 iir = raw_reg_read(regs, GEN8_GT_IIR(0));
0356 if (likely(iir)) {
0357 intel_engine_cs_irq(gt->engine_class[RENDER_CLASS][0],
0358 iir >> GEN8_RCS_IRQ_SHIFT);
0359 intel_engine_cs_irq(gt->engine_class[COPY_ENGINE_CLASS][0],
0360 iir >> GEN8_BCS_IRQ_SHIFT);
0361 raw_reg_write(regs, GEN8_GT_IIR(0), iir);
0362 }
0363 }
0364
0365 if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
0366 iir = raw_reg_read(regs, GEN8_GT_IIR(1));
0367 if (likely(iir)) {
0368 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][0],
0369 iir >> GEN8_VCS0_IRQ_SHIFT);
0370 intel_engine_cs_irq(gt->engine_class[VIDEO_DECODE_CLASS][1],
0371 iir >> GEN8_VCS1_IRQ_SHIFT);
0372 raw_reg_write(regs, GEN8_GT_IIR(1), iir);
0373 }
0374 }
0375
0376 if (master_ctl & GEN8_GT_VECS_IRQ) {
0377 iir = raw_reg_read(regs, GEN8_GT_IIR(3));
0378 if (likely(iir)) {
0379 intel_engine_cs_irq(gt->engine_class[VIDEO_ENHANCEMENT_CLASS][0],
0380 iir >> GEN8_VECS_IRQ_SHIFT);
0381 raw_reg_write(regs, GEN8_GT_IIR(3), iir);
0382 }
0383 }
0384
0385 if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
0386 iir = raw_reg_read(regs, GEN8_GT_IIR(2));
0387 if (likely(iir)) {
0388 gen6_rps_irq_handler(>->rps, iir);
0389 guc_irq_handler(>->uc.guc, iir >> 16);
0390 raw_reg_write(regs, GEN8_GT_IIR(2), iir);
0391 }
0392 }
0393 }
0394
0395 void gen8_gt_irq_reset(struct intel_gt *gt)
0396 {
0397 struct intel_uncore *uncore = gt->uncore;
0398
0399 GEN8_IRQ_RESET_NDX(uncore, GT, 0);
0400 GEN8_IRQ_RESET_NDX(uncore, GT, 1);
0401 GEN8_IRQ_RESET_NDX(uncore, GT, 2);
0402 GEN8_IRQ_RESET_NDX(uncore, GT, 3);
0403 }
0404
0405 void gen8_gt_irq_postinstall(struct intel_gt *gt)
0406 {
0407
0408 const u32 irqs =
0409 GT_CS_MASTER_ERROR_INTERRUPT |
0410 GT_RENDER_USER_INTERRUPT |
0411 GT_CONTEXT_SWITCH_INTERRUPT |
0412 GT_WAIT_SEMAPHORE_INTERRUPT;
0413 const u32 gt_interrupts[] = {
0414 irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT,
0415 irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT,
0416 0,
0417 irqs << GEN8_VECS_IRQ_SHIFT,
0418 };
0419 struct intel_uncore *uncore = gt->uncore;
0420
0421 gt->pm_ier = 0x0;
0422 gt->pm_imr = ~gt->pm_ier;
0423 GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
0424 GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
0425
0426
0427
0428
0429 GEN8_IRQ_INIT_NDX(uncore, GT, 2, gt->pm_imr, gt->pm_ier);
0430 GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
0431 }
0432
0433 static void gen5_gt_update_irq(struct intel_gt *gt,
0434 u32 interrupt_mask,
0435 u32 enabled_irq_mask)
0436 {
0437 lockdep_assert_held(>->irq_lock);
0438
0439 GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
0440
0441 gt->gt_imr &= ~interrupt_mask;
0442 gt->gt_imr |= (~enabled_irq_mask & interrupt_mask);
0443 intel_uncore_write(gt->uncore, GTIMR, gt->gt_imr);
0444 }
0445
0446 void gen5_gt_enable_irq(struct intel_gt *gt, u32 mask)
0447 {
0448 gen5_gt_update_irq(gt, mask, mask);
0449 intel_uncore_posting_read_fw(gt->uncore, GTIMR);
0450 }
0451
0452 void gen5_gt_disable_irq(struct intel_gt *gt, u32 mask)
0453 {
0454 gen5_gt_update_irq(gt, mask, 0);
0455 }
0456
0457 void gen5_gt_irq_reset(struct intel_gt *gt)
0458 {
0459 struct intel_uncore *uncore = gt->uncore;
0460
0461 GEN3_IRQ_RESET(uncore, GT);
0462 if (GRAPHICS_VER(gt->i915) >= 6)
0463 GEN3_IRQ_RESET(uncore, GEN6_PM);
0464 }
0465
0466 void gen5_gt_irq_postinstall(struct intel_gt *gt)
0467 {
0468 struct intel_uncore *uncore = gt->uncore;
0469 u32 pm_irqs = 0;
0470 u32 gt_irqs = 0;
0471
0472 gt->gt_imr = ~0;
0473 if (HAS_L3_DPF(gt->i915)) {
0474
0475 gt->gt_imr = ~GT_PARITY_ERROR(gt->i915);
0476 gt_irqs |= GT_PARITY_ERROR(gt->i915);
0477 }
0478
0479 gt_irqs |= GT_RENDER_USER_INTERRUPT;
0480 if (GRAPHICS_VER(gt->i915) == 5)
0481 gt_irqs |= ILK_BSD_USER_INTERRUPT;
0482 else
0483 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
0484
0485 GEN3_IRQ_INIT(uncore, GT, gt->gt_imr, gt_irqs);
0486
0487 if (GRAPHICS_VER(gt->i915) >= 6) {
0488
0489
0490
0491
0492 if (HAS_ENGINE(gt, VECS0)) {
0493 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
0494 gt->pm_ier |= PM_VEBOX_USER_INTERRUPT;
0495 }
0496
0497 gt->pm_imr = 0xffffffff;
0498 GEN3_IRQ_INIT(uncore, GEN6_PM, gt->pm_imr, pm_irqs);
0499 }
0500 }