Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2019 Intel Corporation
0004  */
0005 
0006 #include <drm/drm_managed.h>
0007 #include <drm/intel-gtt.h>
0008 
0009 #include "gem/i915_gem_internal.h"
0010 #include "gem/i915_gem_lmem.h"
0011 #include "pxp/intel_pxp.h"
0012 
0013 #include "i915_drv.h"
0014 #include "i915_perf_oa_regs.h"
0015 #include "intel_context.h"
0016 #include "intel_engine_pm.h"
0017 #include "intel_engine_regs.h"
0018 #include "intel_ggtt_gmch.h"
0019 #include "intel_gt.h"
0020 #include "intel_gt_buffer_pool.h"
0021 #include "intel_gt_clock_utils.h"
0022 #include "intel_gt_debugfs.h"
0023 #include "intel_gt_mcr.h"
0024 #include "intel_gt_pm.h"
0025 #include "intel_gt_regs.h"
0026 #include "intel_gt_requests.h"
0027 #include "intel_migrate.h"
0028 #include "intel_mocs.h"
0029 #include "intel_pm.h"
0030 #include "intel_rc6.h"
0031 #include "intel_renderstate.h"
0032 #include "intel_rps.h"
0033 #include "intel_gt_sysfs.h"
0034 #include "intel_uncore.h"
0035 #include "shmem_utils.h"
0036 
0037 static void __intel_gt_init_early(struct intel_gt *gt)
0038 {
0039     spin_lock_init(&gt->irq_lock);
0040 
0041     INIT_LIST_HEAD(&gt->closed_vma);
0042     spin_lock_init(&gt->closed_lock);
0043 
0044     init_llist_head(&gt->watchdog.list);
0045     INIT_WORK(&gt->watchdog.work, intel_gt_watchdog_work);
0046 
0047     intel_gt_init_buffer_pool(gt);
0048     intel_gt_init_reset(gt);
0049     intel_gt_init_requests(gt);
0050     intel_gt_init_timelines(gt);
0051     mutex_init(&gt->tlb.invalidate_lock);
0052     seqcount_mutex_init(&gt->tlb.seqno, &gt->tlb.invalidate_lock);
0053     intel_gt_pm_init_early(gt);
0054 
0055     intel_uc_init_early(&gt->uc);
0056     intel_rps_init_early(&gt->rps);
0057 }
0058 
0059 /* Preliminary initialization of Tile 0 */
0060 void intel_root_gt_init_early(struct drm_i915_private *i915)
0061 {
0062     struct intel_gt *gt = to_gt(i915);
0063 
0064     gt->i915 = i915;
0065     gt->uncore = &i915->uncore;
0066 
0067     __intel_gt_init_early(gt);
0068 }
0069 
0070 static int intel_gt_probe_lmem(struct intel_gt *gt)
0071 {
0072     struct drm_i915_private *i915 = gt->i915;
0073     unsigned int instance = gt->info.id;
0074     int id = INTEL_REGION_LMEM_0 + instance;
0075     struct intel_memory_region *mem;
0076     int err;
0077 
0078     mem = intel_gt_setup_lmem(gt);
0079     if (IS_ERR(mem)) {
0080         err = PTR_ERR(mem);
0081         if (err == -ENODEV)
0082             return 0;
0083 
0084         drm_err(&i915->drm,
0085             "Failed to setup region(%d) type=%d\n",
0086             err, INTEL_MEMORY_LOCAL);
0087         return err;
0088     }
0089 
0090     mem->id = id;
0091     mem->instance = instance;
0092 
0093     intel_memory_region_set_name(mem, "local%u", mem->instance);
0094 
0095     GEM_BUG_ON(!HAS_REGION(i915, id));
0096     GEM_BUG_ON(i915->mm.regions[id]);
0097     i915->mm.regions[id] = mem;
0098 
0099     return 0;
0100 }
0101 
0102 int intel_gt_assign_ggtt(struct intel_gt *gt)
0103 {
0104     gt->ggtt = drmm_kzalloc(&gt->i915->drm, sizeof(*gt->ggtt), GFP_KERNEL);
0105 
0106     return gt->ggtt ? 0 : -ENOMEM;
0107 }
0108 
0109 int intel_gt_init_mmio(struct intel_gt *gt)
0110 {
0111     intel_gt_init_clock_frequency(gt);
0112 
0113     intel_uc_init_mmio(&gt->uc);
0114     intel_sseu_info_init(gt);
0115     intel_gt_mcr_init(gt);
0116 
0117     return intel_engines_init_mmio(gt);
0118 }
0119 
0120 static void init_unused_ring(struct intel_gt *gt, u32 base)
0121 {
0122     struct intel_uncore *uncore = gt->uncore;
0123 
0124     intel_uncore_write(uncore, RING_CTL(base), 0);
0125     intel_uncore_write(uncore, RING_HEAD(base), 0);
0126     intel_uncore_write(uncore, RING_TAIL(base), 0);
0127     intel_uncore_write(uncore, RING_START(base), 0);
0128 }
0129 
0130 static void init_unused_rings(struct intel_gt *gt)
0131 {
0132     struct drm_i915_private *i915 = gt->i915;
0133 
0134     if (IS_I830(i915)) {
0135         init_unused_ring(gt, PRB1_BASE);
0136         init_unused_ring(gt, SRB0_BASE);
0137         init_unused_ring(gt, SRB1_BASE);
0138         init_unused_ring(gt, SRB2_BASE);
0139         init_unused_ring(gt, SRB3_BASE);
0140     } else if (GRAPHICS_VER(i915) == 2) {
0141         init_unused_ring(gt, SRB0_BASE);
0142         init_unused_ring(gt, SRB1_BASE);
0143     } else if (GRAPHICS_VER(i915) == 3) {
0144         init_unused_ring(gt, PRB1_BASE);
0145         init_unused_ring(gt, PRB2_BASE);
0146     }
0147 }
0148 
0149 int intel_gt_init_hw(struct intel_gt *gt)
0150 {
0151     struct drm_i915_private *i915 = gt->i915;
0152     struct intel_uncore *uncore = gt->uncore;
0153     int ret;
0154 
0155     gt->last_init_time = ktime_get();
0156 
0157     /* Double layer security blanket, see i915_gem_init() */
0158     intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
0159 
0160     if (HAS_EDRAM(i915) && GRAPHICS_VER(i915) < 9)
0161         intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf));
0162 
0163     if (IS_HASWELL(i915))
0164         intel_uncore_write(uncore,
0165                    HSW_MI_PREDICATE_RESULT_2,
0166                    IS_HSW_GT3(i915) ?
0167                    LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
0168 
0169     /* Apply the GT workarounds... */
0170     intel_gt_apply_workarounds(gt);
0171     /* ...and determine whether they are sticking. */
0172     intel_gt_verify_workarounds(gt, "init");
0173 
0174     intel_gt_init_swizzling(gt);
0175 
0176     /*
0177      * At least 830 can leave some of the unused rings
0178      * "active" (ie. head != tail) after resume which
0179      * will prevent c3 entry. Makes sure all unused rings
0180      * are totally idle.
0181      */
0182     init_unused_rings(gt);
0183 
0184     ret = i915_ppgtt_init_hw(gt);
0185     if (ret) {
0186         DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
0187         goto out;
0188     }
0189 
0190     /* We can't enable contexts until all firmware is loaded */
0191     ret = intel_uc_init_hw(&gt->uc);
0192     if (ret) {
0193         i915_probe_error(i915, "Enabling uc failed (%d)\n", ret);
0194         goto out;
0195     }
0196 
0197     intel_mocs_init(gt);
0198 
0199 out:
0200     intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
0201     return ret;
0202 }
0203 
0204 static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
0205 {
0206     intel_uncore_rmw(uncore, reg, 0, set);
0207 }
0208 
0209 static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
0210 {
0211     intel_uncore_rmw(uncore, reg, clr, 0);
0212 }
0213 
0214 static void clear_register(struct intel_uncore *uncore, i915_reg_t reg)
0215 {
0216     intel_uncore_rmw(uncore, reg, 0, 0);
0217 }
0218 
0219 static void gen6_clear_engine_error_register(struct intel_engine_cs *engine)
0220 {
0221     GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0);
0222     GEN6_RING_FAULT_REG_POSTING_READ(engine);
0223 }
0224 
0225 void
0226 intel_gt_clear_error_registers(struct intel_gt *gt,
0227                    intel_engine_mask_t engine_mask)
0228 {
0229     struct drm_i915_private *i915 = gt->i915;
0230     struct intel_uncore *uncore = gt->uncore;
0231     u32 eir;
0232 
0233     if (GRAPHICS_VER(i915) != 2)
0234         clear_register(uncore, PGTBL_ER);
0235 
0236     if (GRAPHICS_VER(i915) < 4)
0237         clear_register(uncore, IPEIR(RENDER_RING_BASE));
0238     else
0239         clear_register(uncore, IPEIR_I965);
0240 
0241     clear_register(uncore, EIR);
0242     eir = intel_uncore_read(uncore, EIR);
0243     if (eir) {
0244         /*
0245          * some errors might have become stuck,
0246          * mask them.
0247          */
0248         DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
0249         rmw_set(uncore, EMR, eir);
0250         intel_uncore_write(uncore, GEN2_IIR,
0251                    I915_MASTER_ERROR_INTERRUPT);
0252     }
0253 
0254     if (GRAPHICS_VER(i915) >= 12) {
0255         rmw_clear(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID);
0256         intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG);
0257     } else if (GRAPHICS_VER(i915) >= 8) {
0258         rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID);
0259         intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);
0260     } else if (GRAPHICS_VER(i915) >= 6) {
0261         struct intel_engine_cs *engine;
0262         enum intel_engine_id id;
0263 
0264         for_each_engine_masked(engine, gt, engine_mask, id)
0265             gen6_clear_engine_error_register(engine);
0266     }
0267 }
0268 
0269 static void gen6_check_faults(struct intel_gt *gt)
0270 {
0271     struct intel_engine_cs *engine;
0272     enum intel_engine_id id;
0273     u32 fault;
0274 
0275     for_each_engine(engine, gt, id) {
0276         fault = GEN6_RING_FAULT_REG_READ(engine);
0277         if (fault & RING_FAULT_VALID) {
0278             drm_dbg(&engine->i915->drm, "Unexpected fault\n"
0279                 "\tAddr: 0x%08lx\n"
0280                 "\tAddress space: %s\n"
0281                 "\tSource ID: %d\n"
0282                 "\tType: %d\n",
0283                 fault & PAGE_MASK,
0284                 fault & RING_FAULT_GTTSEL_MASK ?
0285                 "GGTT" : "PPGTT",
0286                 RING_FAULT_SRCID(fault),
0287                 RING_FAULT_FAULT_TYPE(fault));
0288         }
0289     }
0290 }
0291 
0292 static void gen8_check_faults(struct intel_gt *gt)
0293 {
0294     struct intel_uncore *uncore = gt->uncore;
0295     i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg;
0296     u32 fault;
0297 
0298     if (GRAPHICS_VER(gt->i915) >= 12) {
0299         fault_reg = GEN12_RING_FAULT_REG;
0300         fault_data0_reg = GEN12_FAULT_TLB_DATA0;
0301         fault_data1_reg = GEN12_FAULT_TLB_DATA1;
0302     } else {
0303         fault_reg = GEN8_RING_FAULT_REG;
0304         fault_data0_reg = GEN8_FAULT_TLB_DATA0;
0305         fault_data1_reg = GEN8_FAULT_TLB_DATA1;
0306     }
0307 
0308     fault = intel_uncore_read(uncore, fault_reg);
0309     if (fault & RING_FAULT_VALID) {
0310         u32 fault_data0, fault_data1;
0311         u64 fault_addr;
0312 
0313         fault_data0 = intel_uncore_read(uncore, fault_data0_reg);
0314         fault_data1 = intel_uncore_read(uncore, fault_data1_reg);
0315 
0316         fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
0317                  ((u64)fault_data0 << 12);
0318 
0319         drm_dbg(&uncore->i915->drm, "Unexpected fault\n"
0320             "\tAddr: 0x%08x_%08x\n"
0321             "\tAddress space: %s\n"
0322             "\tEngine ID: %d\n"
0323             "\tSource ID: %d\n"
0324             "\tType: %d\n",
0325             upper_32_bits(fault_addr), lower_32_bits(fault_addr),
0326             fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
0327             GEN8_RING_FAULT_ENGINE_ID(fault),
0328             RING_FAULT_SRCID(fault),
0329             RING_FAULT_FAULT_TYPE(fault));
0330     }
0331 }
0332 
0333 void intel_gt_check_and_clear_faults(struct intel_gt *gt)
0334 {
0335     struct drm_i915_private *i915 = gt->i915;
0336 
0337     /* From GEN8 onwards we only have one 'All Engine Fault Register' */
0338     if (GRAPHICS_VER(i915) >= 8)
0339         gen8_check_faults(gt);
0340     else if (GRAPHICS_VER(i915) >= 6)
0341         gen6_check_faults(gt);
0342     else
0343         return;
0344 
0345     intel_gt_clear_error_registers(gt, ALL_ENGINES);
0346 }
0347 
0348 void intel_gt_flush_ggtt_writes(struct intel_gt *gt)
0349 {
0350     struct intel_uncore *uncore = gt->uncore;
0351     intel_wakeref_t wakeref;
0352 
0353     /*
0354      * No actual flushing is required for the GTT write domain for reads
0355      * from the GTT domain. Writes to it "immediately" go to main memory
0356      * as far as we know, so there's no chipset flush. It also doesn't
0357      * land in the GPU render cache.
0358      *
0359      * However, we do have to enforce the order so that all writes through
0360      * the GTT land before any writes to the device, such as updates to
0361      * the GATT itself.
0362      *
0363      * We also have to wait a bit for the writes to land from the GTT.
0364      * An uncached read (i.e. mmio) seems to be ideal for the round-trip
0365      * timing. This issue has only been observed when switching quickly
0366      * between GTT writes and CPU reads from inside the kernel on recent hw,
0367      * and it appears to only affect discrete GTT blocks (i.e. on LLC
0368      * system agents we cannot reproduce this behaviour, until Cannonlake
0369      * that was!).
0370      */
0371 
0372     wmb();
0373 
0374     if (INTEL_INFO(gt->i915)->has_coherent_ggtt)
0375         return;
0376 
0377     intel_gt_chipset_flush(gt);
0378 
0379     with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) {
0380         unsigned long flags;
0381 
0382         spin_lock_irqsave(&uncore->lock, flags);
0383         intel_uncore_posting_read_fw(uncore,
0384                          RING_HEAD(RENDER_RING_BASE));
0385         spin_unlock_irqrestore(&uncore->lock, flags);
0386     }
0387 }
0388 
0389 void intel_gt_chipset_flush(struct intel_gt *gt)
0390 {
0391     wmb();
0392     if (GRAPHICS_VER(gt->i915) < 6)
0393         intel_ggtt_gmch_flush();
0394 }
0395 
0396 void intel_gt_driver_register(struct intel_gt *gt)
0397 {
0398     intel_gsc_init(&gt->gsc, gt->i915);
0399 
0400     intel_rps_driver_register(&gt->rps);
0401 
0402     intel_gt_debugfs_register(gt);
0403     intel_gt_sysfs_register(gt);
0404 }
0405 
0406 static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
0407 {
0408     struct drm_i915_private *i915 = gt->i915;
0409     struct drm_i915_gem_object *obj;
0410     struct i915_vma *vma;
0411     int ret;
0412 
0413     obj = i915_gem_object_create_lmem(i915, size,
0414                       I915_BO_ALLOC_VOLATILE |
0415                       I915_BO_ALLOC_GPU_ONLY);
0416     if (IS_ERR(obj))
0417         obj = i915_gem_object_create_stolen(i915, size);
0418     if (IS_ERR(obj))
0419         obj = i915_gem_object_create_internal(i915, size);
0420     if (IS_ERR(obj)) {
0421         drm_err(&i915->drm, "Failed to allocate scratch page\n");
0422         return PTR_ERR(obj);
0423     }
0424 
0425     vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
0426     if (IS_ERR(vma)) {
0427         ret = PTR_ERR(vma);
0428         goto err_unref;
0429     }
0430 
0431     ret = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
0432     if (ret)
0433         goto err_unref;
0434 
0435     gt->scratch = i915_vma_make_unshrinkable(vma);
0436 
0437     return 0;
0438 
0439 err_unref:
0440     i915_gem_object_put(obj);
0441     return ret;
0442 }
0443 
0444 static void intel_gt_fini_scratch(struct intel_gt *gt)
0445 {
0446     i915_vma_unpin_and_release(&gt->scratch, 0);
0447 }
0448 
0449 static struct i915_address_space *kernel_vm(struct intel_gt *gt)
0450 {
0451     if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING)
0452         return &i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY)->vm;
0453     else
0454         return i915_vm_get(&gt->ggtt->vm);
0455 }
0456 
0457 static int __engines_record_defaults(struct intel_gt *gt)
0458 {
0459     struct i915_request *requests[I915_NUM_ENGINES] = {};
0460     struct intel_engine_cs *engine;
0461     enum intel_engine_id id;
0462     int err = 0;
0463 
0464     /*
0465      * As we reset the gpu during very early sanitisation, the current
0466      * register state on the GPU should reflect its defaults values.
0467      * We load a context onto the hw (with restore-inhibit), then switch
0468      * over to a second context to save that default register state. We
0469      * can then prime every new context with that state so they all start
0470      * from the same default HW values.
0471      */
0472 
0473     for_each_engine(engine, gt, id) {
0474         struct intel_renderstate so;
0475         struct intel_context *ce;
0476         struct i915_request *rq;
0477 
0478         /* We must be able to switch to something! */
0479         GEM_BUG_ON(!engine->kernel_context);
0480 
0481         ce = intel_context_create(engine);
0482         if (IS_ERR(ce)) {
0483             err = PTR_ERR(ce);
0484             goto out;
0485         }
0486 
0487         err = intel_renderstate_init(&so, ce);
0488         if (err)
0489             goto err;
0490 
0491         rq = i915_request_create(ce);
0492         if (IS_ERR(rq)) {
0493             err = PTR_ERR(rq);
0494             goto err_fini;
0495         }
0496 
0497         err = intel_engine_emit_ctx_wa(rq);
0498         if (err)
0499             goto err_rq;
0500 
0501         err = intel_renderstate_emit(&so, rq);
0502         if (err)
0503             goto err_rq;
0504 
0505 err_rq:
0506         requests[id] = i915_request_get(rq);
0507         i915_request_add(rq);
0508 err_fini:
0509         intel_renderstate_fini(&so, ce);
0510 err:
0511         if (err) {
0512             intel_context_put(ce);
0513             goto out;
0514         }
0515     }
0516 
0517     /* Flush the default context image to memory, and enable powersaving. */
0518     if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
0519         err = -EIO;
0520         goto out;
0521     }
0522 
0523     for (id = 0; id < ARRAY_SIZE(requests); id++) {
0524         struct i915_request *rq;
0525         struct file *state;
0526 
0527         rq = requests[id];
0528         if (!rq)
0529             continue;
0530 
0531         if (rq->fence.error) {
0532             err = -EIO;
0533             goto out;
0534         }
0535 
0536         GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags));
0537         if (!rq->context->state)
0538             continue;
0539 
0540         /* Keep a copy of the state's backing pages; free the obj */
0541         state = shmem_create_from_object(rq->context->state->obj);
0542         if (IS_ERR(state)) {
0543             err = PTR_ERR(state);
0544             goto out;
0545         }
0546         rq->engine->default_state = state;
0547     }
0548 
0549 out:
0550     /*
0551      * If we have to abandon now, we expect the engines to be idle
0552      * and ready to be torn-down. The quickest way we can accomplish
0553      * this is by declaring ourselves wedged.
0554      */
0555     if (err)
0556         intel_gt_set_wedged(gt);
0557 
0558     for (id = 0; id < ARRAY_SIZE(requests); id++) {
0559         struct intel_context *ce;
0560         struct i915_request *rq;
0561 
0562         rq = requests[id];
0563         if (!rq)
0564             continue;
0565 
0566         ce = rq->context;
0567         i915_request_put(rq);
0568         intel_context_put(ce);
0569     }
0570     return err;
0571 }
0572 
0573 static int __engines_verify_workarounds(struct intel_gt *gt)
0574 {
0575     struct intel_engine_cs *engine;
0576     enum intel_engine_id id;
0577     int err = 0;
0578 
0579     if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
0580         return 0;
0581 
0582     for_each_engine(engine, gt, id) {
0583         if (intel_engine_verify_workarounds(engine, "load"))
0584             err = -EIO;
0585     }
0586 
0587     /* Flush and restore the kernel context for safety */
0588     if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME)
0589         err = -EIO;
0590 
0591     return err;
0592 }
0593 
0594 static void __intel_gt_disable(struct intel_gt *gt)
0595 {
0596     intel_gt_set_wedged_on_fini(gt);
0597 
0598     intel_gt_suspend_prepare(gt);
0599     intel_gt_suspend_late(gt);
0600 
0601     GEM_BUG_ON(intel_gt_pm_is_awake(gt));
0602 }
0603 
0604 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout)
0605 {
0606     long remaining_timeout;
0607 
0608     /* If the device is asleep, we have no requests outstanding */
0609     if (!intel_gt_pm_is_awake(gt))
0610         return 0;
0611 
0612     while ((timeout = intel_gt_retire_requests_timeout(gt, timeout,
0613                                &remaining_timeout)) > 0) {
0614         cond_resched();
0615         if (signal_pending(current))
0616             return -EINTR;
0617     }
0618 
0619     return timeout ? timeout : intel_uc_wait_for_idle(&gt->uc,
0620                               remaining_timeout);
0621 }
0622 
0623 int intel_gt_init(struct intel_gt *gt)
0624 {
0625     int err;
0626 
0627     err = i915_inject_probe_error(gt->i915, -ENODEV);
0628     if (err)
0629         return err;
0630 
0631     intel_gt_init_workarounds(gt);
0632 
0633     /*
0634      * This is just a security blanket to placate dragons.
0635      * On some systems, we very sporadically observe that the first TLBs
0636      * used by the CS may be stale, despite us poking the TLB reset. If
0637      * we hold the forcewake during initialisation these problems
0638      * just magically go away.
0639      */
0640     intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
0641 
0642     err = intel_gt_init_scratch(gt,
0643                     GRAPHICS_VER(gt->i915) == 2 ? SZ_256K : SZ_4K);
0644     if (err)
0645         goto out_fw;
0646 
0647     intel_gt_pm_init(gt);
0648 
0649     gt->vm = kernel_vm(gt);
0650     if (!gt->vm) {
0651         err = -ENOMEM;
0652         goto err_pm;
0653     }
0654 
0655     intel_set_mocs_index(gt);
0656 
0657     err = intel_engines_init(gt);
0658     if (err)
0659         goto err_engines;
0660 
0661     err = intel_uc_init(&gt->uc);
0662     if (err)
0663         goto err_engines;
0664 
0665     err = intel_gt_resume(gt);
0666     if (err)
0667         goto err_uc_init;
0668 
0669     err = intel_gt_init_hwconfig(gt);
0670     if (err)
0671         drm_err(&gt->i915->drm, "Failed to retrieve hwconfig table: %pe\n",
0672             ERR_PTR(err));
0673 
0674     err = __engines_record_defaults(gt);
0675     if (err)
0676         goto err_gt;
0677 
0678     err = __engines_verify_workarounds(gt);
0679     if (err)
0680         goto err_gt;
0681 
0682     intel_uc_init_late(&gt->uc);
0683 
0684     err = i915_inject_probe_error(gt->i915, -EIO);
0685     if (err)
0686         goto err_gt;
0687 
0688     intel_migrate_init(&gt->migrate, gt);
0689 
0690     intel_pxp_init(&gt->pxp);
0691 
0692     goto out_fw;
0693 err_gt:
0694     __intel_gt_disable(gt);
0695     intel_uc_fini_hw(&gt->uc);
0696 err_uc_init:
0697     intel_uc_fini(&gt->uc);
0698 err_engines:
0699     intel_engines_release(gt);
0700     i915_vm_put(fetch_and_zero(&gt->vm));
0701 err_pm:
0702     intel_gt_pm_fini(gt);
0703     intel_gt_fini_scratch(gt);
0704 out_fw:
0705     if (err)
0706         intel_gt_set_wedged_on_init(gt);
0707     intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
0708     return err;
0709 }
0710 
0711 void intel_gt_driver_remove(struct intel_gt *gt)
0712 {
0713     __intel_gt_disable(gt);
0714 
0715     intel_migrate_fini(&gt->migrate);
0716     intel_uc_driver_remove(&gt->uc);
0717 
0718     intel_engines_release(gt);
0719 
0720     intel_gt_flush_buffer_pool(gt);
0721 }
0722 
0723 void intel_gt_driver_unregister(struct intel_gt *gt)
0724 {
0725     intel_wakeref_t wakeref;
0726 
0727     intel_gt_sysfs_unregister(gt);
0728     intel_rps_driver_unregister(&gt->rps);
0729     intel_gsc_fini(&gt->gsc);
0730 
0731     intel_pxp_fini(&gt->pxp);
0732 
0733     /*
0734      * Upon unregistering the device to prevent any new users, cancel
0735      * all in-flight requests so that we can quickly unbind the active
0736      * resources.
0737      */
0738     intel_gt_set_wedged_on_fini(gt);
0739 
0740     /* Scrub all HW state upon release */
0741     with_intel_runtime_pm(gt->uncore->rpm, wakeref)
0742         __intel_gt_reset(gt, ALL_ENGINES);
0743 }
0744 
0745 void intel_gt_driver_release(struct intel_gt *gt)
0746 {
0747     struct i915_address_space *vm;
0748 
0749     vm = fetch_and_zero(&gt->vm);
0750     if (vm) /* FIXME being called twice on error paths :( */
0751         i915_vm_put(vm);
0752 
0753     intel_wa_list_free(&gt->wa_list);
0754     intel_gt_pm_fini(gt);
0755     intel_gt_fini_scratch(gt);
0756     intel_gt_fini_buffer_pool(gt);
0757     intel_gt_fini_hwconfig(gt);
0758 }
0759 
0760 void intel_gt_driver_late_release_all(struct drm_i915_private *i915)
0761 {
0762     struct intel_gt *gt;
0763     unsigned int id;
0764 
0765     /* We need to wait for inflight RCU frees to release their grip */
0766     rcu_barrier();
0767 
0768     for_each_gt(gt, i915, id) {
0769         intel_uc_driver_late_release(&gt->uc);
0770         intel_gt_fini_requests(gt);
0771         intel_gt_fini_reset(gt);
0772         intel_gt_fini_timelines(gt);
0773         mutex_destroy(&gt->tlb.invalidate_lock);
0774         intel_engines_free(gt);
0775     }
0776 }
0777 
0778 static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr)
0779 {
0780     int ret;
0781 
0782     if (!gt_is_root(gt)) {
0783         struct intel_uncore_mmio_debug *mmio_debug;
0784         struct intel_uncore *uncore;
0785 
0786         uncore = kzalloc(sizeof(*uncore), GFP_KERNEL);
0787         if (!uncore)
0788             return -ENOMEM;
0789 
0790         mmio_debug = kzalloc(sizeof(*mmio_debug), GFP_KERNEL);
0791         if (!mmio_debug) {
0792             kfree(uncore);
0793             return -ENOMEM;
0794         }
0795 
0796         gt->uncore = uncore;
0797         gt->uncore->debug = mmio_debug;
0798 
0799         __intel_gt_init_early(gt);
0800     }
0801 
0802     intel_uncore_init_early(gt->uncore, gt);
0803 
0804     ret = intel_uncore_setup_mmio(gt->uncore, phys_addr);
0805     if (ret)
0806         return ret;
0807 
0808     gt->phys_addr = phys_addr;
0809 
0810     return 0;
0811 }
0812 
0813 static void
0814 intel_gt_tile_cleanup(struct intel_gt *gt)
0815 {
0816     intel_uncore_cleanup_mmio(gt->uncore);
0817 
0818     if (!gt_is_root(gt)) {
0819         kfree(gt->uncore->debug);
0820         kfree(gt->uncore);
0821         kfree(gt);
0822     }
0823 }
0824 
0825 int intel_gt_probe_all(struct drm_i915_private *i915)
0826 {
0827     struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
0828     struct intel_gt *gt = &i915->gt0;
0829     phys_addr_t phys_addr;
0830     unsigned int mmio_bar;
0831     int ret;
0832 
0833     mmio_bar = GRAPHICS_VER(i915) == 2 ? 1 : 0;
0834     phys_addr = pci_resource_start(pdev, mmio_bar);
0835 
0836     /*
0837      * We always have at least one primary GT on any device
0838      * and it has been already initialized early during probe
0839      * in i915_driver_probe()
0840      */
0841     ret = intel_gt_tile_setup(gt, phys_addr);
0842     if (ret)
0843         return ret;
0844 
0845     i915->gt[0] = gt;
0846 
0847     /* TODO: add more tiles */
0848     return 0;
0849 }
0850 
0851 int intel_gt_tiles_init(struct drm_i915_private *i915)
0852 {
0853     struct intel_gt *gt;
0854     unsigned int id;
0855     int ret;
0856 
0857     for_each_gt(gt, i915, id) {
0858         ret = intel_gt_probe_lmem(gt);
0859         if (ret)
0860             return ret;
0861     }
0862 
0863     return 0;
0864 }
0865 
0866 void intel_gt_release_all(struct drm_i915_private *i915)
0867 {
0868     struct intel_gt *gt;
0869     unsigned int id;
0870 
0871     for_each_gt(gt, i915, id) {
0872         intel_gt_tile_cleanup(gt);
0873         i915->gt[id] = NULL;
0874     }
0875 }
0876 
0877 void intel_gt_info_print(const struct intel_gt_info *info,
0878              struct drm_printer *p)
0879 {
0880     drm_printf(p, "available engines: %x\n", info->engine_mask);
0881 
0882     intel_sseu_dump(&info->sseu, p);
0883 }
0884 
0885 struct reg_and_bit {
0886     i915_reg_t reg;
0887     u32 bit;
0888 };
0889 
0890 static struct reg_and_bit
0891 get_reg_and_bit(const struct intel_engine_cs *engine, const bool gen8,
0892         const i915_reg_t *regs, const unsigned int num)
0893 {
0894     const unsigned int class = engine->class;
0895     struct reg_and_bit rb = { };
0896 
0897     if (drm_WARN_ON_ONCE(&engine->i915->drm,
0898                  class >= num || !regs[class].reg))
0899         return rb;
0900 
0901     rb.reg = regs[class];
0902     if (gen8 && class == VIDEO_DECODE_CLASS)
0903         rb.reg.reg += 4 * engine->instance; /* GEN8_M2TCR */
0904     else
0905         rb.bit = engine->instance;
0906 
0907     rb.bit = BIT(rb.bit);
0908 
0909     return rb;
0910 }
0911 
0912 static void mmio_invalidate_full(struct intel_gt *gt)
0913 {
0914     static const i915_reg_t gen8_regs[] = {
0915         [RENDER_CLASS]          = GEN8_RTCR,
0916         [VIDEO_DECODE_CLASS]        = GEN8_M1TCR, /* , GEN8_M2TCR */
0917         [VIDEO_ENHANCEMENT_CLASS]   = GEN8_VTCR,
0918         [COPY_ENGINE_CLASS]     = GEN8_BTCR,
0919     };
0920     static const i915_reg_t gen12_regs[] = {
0921         [RENDER_CLASS]          = GEN12_GFX_TLB_INV_CR,
0922         [VIDEO_DECODE_CLASS]        = GEN12_VD_TLB_INV_CR,
0923         [VIDEO_ENHANCEMENT_CLASS]   = GEN12_VE_TLB_INV_CR,
0924         [COPY_ENGINE_CLASS]     = GEN12_BLT_TLB_INV_CR,
0925         [COMPUTE_CLASS]         = GEN12_COMPCTX_TLB_INV_CR,
0926     };
0927     struct drm_i915_private *i915 = gt->i915;
0928     struct intel_uncore *uncore = gt->uncore;
0929     struct intel_engine_cs *engine;
0930     intel_engine_mask_t awake, tmp;
0931     enum intel_engine_id id;
0932     const i915_reg_t *regs;
0933     unsigned int num = 0;
0934 
0935     if (GRAPHICS_VER(i915) == 12) {
0936         regs = gen12_regs;
0937         num = ARRAY_SIZE(gen12_regs);
0938     } else if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) <= 11) {
0939         regs = gen8_regs;
0940         num = ARRAY_SIZE(gen8_regs);
0941     } else if (GRAPHICS_VER(i915) < 8) {
0942         return;
0943     }
0944 
0945     if (drm_WARN_ONCE(&i915->drm, !num,
0946               "Platform does not implement TLB invalidation!"))
0947         return;
0948 
0949     intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
0950 
0951     spin_lock_irq(&uncore->lock); /* serialise invalidate with GT reset */
0952 
0953     awake = 0;
0954     for_each_engine(engine, gt, id) {
0955         struct reg_and_bit rb;
0956 
0957         if (!intel_engine_pm_is_awake(engine))
0958             continue;
0959 
0960         rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
0961         if (!i915_mmio_reg_offset(rb.reg))
0962             continue;
0963 
0964         intel_uncore_write_fw(uncore, rb.reg, rb.bit);
0965         awake |= engine->mask;
0966     }
0967 
0968     GT_TRACE(gt, "invalidated engines %08x\n", awake);
0969 
0970     /* Wa_2207587034:tgl,dg1,rkl,adl-s,adl-p */
0971     if (awake &&
0972         (IS_TIGERLAKE(i915) ||
0973          IS_DG1(i915) ||
0974          IS_ROCKETLAKE(i915) ||
0975          IS_ALDERLAKE_S(i915) ||
0976          IS_ALDERLAKE_P(i915)))
0977         intel_uncore_write_fw(uncore, GEN12_OA_TLB_INV_CR, 1);
0978 
0979     spin_unlock_irq(&uncore->lock);
0980 
0981     for_each_engine_masked(engine, gt, awake, tmp) {
0982         struct reg_and_bit rb;
0983 
0984         /*
0985          * HW architecture suggest typical invalidation time at 40us,
0986          * with pessimistic cases up to 100us and a recommendation to
0987          * cap at 1ms. We go a bit higher just in case.
0988          */
0989         const unsigned int timeout_us = 100;
0990         const unsigned int timeout_ms = 4;
0991 
0992         rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
0993         if (__intel_wait_for_register_fw(uncore,
0994                          rb.reg, rb.bit, 0,
0995                          timeout_us, timeout_ms,
0996                          NULL))
0997             drm_err_ratelimited(&gt->i915->drm,
0998                         "%s TLB invalidation did not complete in %ums!\n",
0999                         engine->name, timeout_ms);
1000     }
1001 
1002     /*
1003      * Use delayed put since a) we mostly expect a flurry of TLB
1004      * invalidations so it is good to avoid paying the forcewake cost and
1005      * b) it works around a bug in Icelake which cannot cope with too rapid
1006      * transitions.
1007      */
1008     intel_uncore_forcewake_put_delayed(uncore, FORCEWAKE_ALL);
1009 }
1010 
1011 static bool tlb_seqno_passed(const struct intel_gt *gt, u32 seqno)
1012 {
1013     u32 cur = intel_gt_tlb_seqno(gt);
1014 
1015     /* Only skip if a *full* TLB invalidate barrier has passed */
1016     return (s32)(cur - ALIGN(seqno, 2)) > 0;
1017 }
1018 
1019 void intel_gt_invalidate_tlb(struct intel_gt *gt, u32 seqno)
1020 {
1021     intel_wakeref_t wakeref;
1022 
1023     if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))
1024         return;
1025 
1026     if (intel_gt_is_wedged(gt))
1027         return;
1028 
1029     if (tlb_seqno_passed(gt, seqno))
1030         return;
1031 
1032     with_intel_gt_pm_if_awake(gt, wakeref) {
1033         mutex_lock(&gt->tlb.invalidate_lock);
1034         if (tlb_seqno_passed(gt, seqno))
1035             goto unlock;
1036 
1037         mmio_invalidate_full(gt);
1038 
1039         write_seqcount_invalidate(&gt->tlb.seqno);
1040 unlock:
1041         mutex_unlock(&gt->tlb.invalidate_lock);
1042     }
1043 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!