Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2018 Intel Corporation
0004  */
0005 
0006 #include "gem/i915_gem_internal.h"
0007 #include "gem/i915_gem_pm.h"
0008 #include "gt/intel_engine_user.h"
0009 #include "gt/intel_gt.h"
0010 #include "i915_selftest.h"
0011 #include "intel_reset.h"
0012 
0013 #include "selftests/igt_flush_test.h"
0014 #include "selftests/igt_reset.h"
0015 #include "selftests/igt_spinner.h"
0016 #include "selftests/intel_scheduler_helpers.h"
0017 #include "selftests/mock_drm.h"
0018 
0019 #include "gem/selftests/igt_gem_utils.h"
0020 #include "gem/selftests/mock_context.h"
0021 
0022 static const struct wo_register {
0023     enum intel_platform platform;
0024     u32 reg;
0025 } wo_registers[] = {
0026     { INTEL_GEMINILAKE, 0x731c }
0027 };
0028 
0029 struct wa_lists {
0030     struct i915_wa_list gt_wa_list;
0031     struct {
0032         struct i915_wa_list wa_list;
0033         struct i915_wa_list ctx_wa_list;
0034     } engine[I915_NUM_ENGINES];
0035 };
0036 
0037 static int request_add_sync(struct i915_request *rq, int err)
0038 {
0039     i915_request_get(rq);
0040     i915_request_add(rq);
0041     if (i915_request_wait(rq, 0, HZ / 5) < 0)
0042         err = -EIO;
0043     i915_request_put(rq);
0044 
0045     return err;
0046 }
0047 
0048 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
0049 {
0050     int err = 0;
0051 
0052     i915_request_get(rq);
0053     i915_request_add(rq);
0054     if (spin && !igt_wait_for_spinner(spin, rq))
0055         err = -ETIMEDOUT;
0056     i915_request_put(rq);
0057 
0058     return err;
0059 }
0060 
0061 static void
0062 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
0063 {
0064     struct intel_engine_cs *engine;
0065     enum intel_engine_id id;
0066 
0067     memset(lists, 0, sizeof(*lists));
0068 
0069     wa_init_start(&lists->gt_wa_list, "GT_REF", "global");
0070     gt_init_workarounds(gt, &lists->gt_wa_list);
0071     wa_init_finish(&lists->gt_wa_list);
0072 
0073     for_each_engine(engine, gt, id) {
0074         struct i915_wa_list *wal = &lists->engine[id].wa_list;
0075 
0076         wa_init_start(wal, "REF", engine->name);
0077         engine_init_workarounds(engine, wal);
0078         wa_init_finish(wal);
0079 
0080         __intel_engine_init_ctx_wa(engine,
0081                        &lists->engine[id].ctx_wa_list,
0082                        "CTX_REF");
0083     }
0084 }
0085 
0086 static void
0087 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
0088 {
0089     struct intel_engine_cs *engine;
0090     enum intel_engine_id id;
0091 
0092     for_each_engine(engine, gt, id)
0093         intel_wa_list_free(&lists->engine[id].wa_list);
0094 
0095     intel_wa_list_free(&lists->gt_wa_list);
0096 }
0097 
0098 static struct drm_i915_gem_object *
0099 read_nonprivs(struct intel_context *ce)
0100 {
0101     struct intel_engine_cs *engine = ce->engine;
0102     const u32 base = engine->mmio_base;
0103     struct drm_i915_gem_object *result;
0104     struct i915_request *rq;
0105     struct i915_vma *vma;
0106     u32 srm, *cs;
0107     int err;
0108     int i;
0109 
0110     result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
0111     if (IS_ERR(result))
0112         return result;
0113 
0114     i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
0115 
0116     cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
0117     if (IS_ERR(cs)) {
0118         err = PTR_ERR(cs);
0119         goto err_obj;
0120     }
0121     memset(cs, 0xc5, PAGE_SIZE);
0122     i915_gem_object_flush_map(result);
0123     i915_gem_object_unpin_map(result);
0124 
0125     vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
0126     if (IS_ERR(vma)) {
0127         err = PTR_ERR(vma);
0128         goto err_obj;
0129     }
0130 
0131     err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
0132     if (err)
0133         goto err_obj;
0134 
0135     rq = intel_context_create_request(ce);
0136     if (IS_ERR(rq)) {
0137         err = PTR_ERR(rq);
0138         goto err_pin;
0139     }
0140 
0141     i915_vma_lock(vma);
0142     err = i915_request_await_object(rq, vma->obj, true);
0143     if (err == 0)
0144         err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
0145     i915_vma_unlock(vma);
0146     if (err)
0147         goto err_req;
0148 
0149     srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
0150     if (GRAPHICS_VER(engine->i915) >= 8)
0151         srm++;
0152 
0153     cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
0154     if (IS_ERR(cs)) {
0155         err = PTR_ERR(cs);
0156         goto err_req;
0157     }
0158 
0159     for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
0160         *cs++ = srm;
0161         *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
0162         *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
0163         *cs++ = 0;
0164     }
0165     intel_ring_advance(rq, cs);
0166 
0167     i915_request_add(rq);
0168     i915_vma_unpin(vma);
0169 
0170     return result;
0171 
0172 err_req:
0173     i915_request_add(rq);
0174 err_pin:
0175     i915_vma_unpin(vma);
0176 err_obj:
0177     i915_gem_object_put(result);
0178     return ERR_PTR(err);
0179 }
0180 
0181 static u32
0182 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
0183 {
0184     i915_reg_t reg = i < engine->whitelist.count ?
0185              engine->whitelist.list[i].reg :
0186              RING_NOPID(engine->mmio_base);
0187 
0188     return i915_mmio_reg_offset(reg);
0189 }
0190 
0191 static void
0192 print_results(const struct intel_engine_cs *engine, const u32 *results)
0193 {
0194     unsigned int i;
0195 
0196     for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
0197         u32 expected = get_whitelist_reg(engine, i);
0198         u32 actual = results[i];
0199 
0200         pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
0201             i, expected, actual);
0202     }
0203 }
0204 
0205 static int check_whitelist(struct intel_context *ce)
0206 {
0207     struct intel_engine_cs *engine = ce->engine;
0208     struct drm_i915_gem_object *results;
0209     struct intel_wedge_me wedge;
0210     u32 *vaddr;
0211     int err;
0212     int i;
0213 
0214     results = read_nonprivs(ce);
0215     if (IS_ERR(results))
0216         return PTR_ERR(results);
0217 
0218     err = 0;
0219     i915_gem_object_lock(results, NULL);
0220     intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
0221         err = i915_gem_object_set_to_cpu_domain(results, false);
0222 
0223     if (intel_gt_is_wedged(engine->gt))
0224         err = -EIO;
0225     if (err)
0226         goto out_put;
0227 
0228     vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
0229     if (IS_ERR(vaddr)) {
0230         err = PTR_ERR(vaddr);
0231         goto out_put;
0232     }
0233 
0234     for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
0235         u32 expected = get_whitelist_reg(engine, i);
0236         u32 actual = vaddr[i];
0237 
0238         if (expected != actual) {
0239             print_results(engine, vaddr);
0240             pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
0241                    i, expected, actual);
0242 
0243             err = -EINVAL;
0244             break;
0245         }
0246     }
0247 
0248     i915_gem_object_unpin_map(results);
0249 out_put:
0250     i915_gem_object_unlock(results);
0251     i915_gem_object_put(results);
0252     return err;
0253 }
0254 
0255 static int do_device_reset(struct intel_engine_cs *engine)
0256 {
0257     intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
0258     return 0;
0259 }
0260 
0261 static int do_engine_reset(struct intel_engine_cs *engine)
0262 {
0263     return intel_engine_reset(engine, "live_workarounds");
0264 }
0265 
0266 static int do_guc_reset(struct intel_engine_cs *engine)
0267 {
0268     /* Currently a no-op as the reset is handled by GuC */
0269     return 0;
0270 }
0271 
0272 static int
0273 switch_to_scratch_context(struct intel_engine_cs *engine,
0274               struct igt_spinner *spin,
0275               struct i915_request **rq)
0276 {
0277     struct intel_context *ce;
0278     int err = 0;
0279 
0280     ce = intel_context_create(engine);
0281     if (IS_ERR(ce))
0282         return PTR_ERR(ce);
0283 
0284     *rq = igt_spinner_create_request(spin, ce, MI_NOOP);
0285     intel_context_put(ce);
0286 
0287     if (IS_ERR(*rq)) {
0288         spin = NULL;
0289         err = PTR_ERR(*rq);
0290         goto err;
0291     }
0292 
0293     err = request_add_spin(*rq, spin);
0294 err:
0295     if (err && spin)
0296         igt_spinner_end(spin);
0297 
0298     return err;
0299 }
0300 
0301 static int check_whitelist_across_reset(struct intel_engine_cs *engine,
0302                     int (*reset)(struct intel_engine_cs *),
0303                     const char *name)
0304 {
0305     struct intel_context *ce, *tmp;
0306     struct igt_spinner spin;
0307     struct i915_request *rq;
0308     intel_wakeref_t wakeref;
0309     int err;
0310 
0311     pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
0312         engine->whitelist.count, engine->name, name);
0313 
0314     ce = intel_context_create(engine);
0315     if (IS_ERR(ce))
0316         return PTR_ERR(ce);
0317 
0318     err = igt_spinner_init(&spin, engine->gt);
0319     if (err)
0320         goto out_ctx;
0321 
0322     err = check_whitelist(ce);
0323     if (err) {
0324         pr_err("Invalid whitelist *before* %s reset!\n", name);
0325         goto out_spin;
0326     }
0327 
0328     err = switch_to_scratch_context(engine, &spin, &rq);
0329     if (err)
0330         goto out_spin;
0331 
0332     /* Ensure the spinner hasn't aborted */
0333     if (i915_request_completed(rq)) {
0334         pr_err("%s spinner failed to start\n", name);
0335         err = -ETIMEDOUT;
0336         goto out_spin;
0337     }
0338 
0339     with_intel_runtime_pm(engine->uncore->rpm, wakeref)
0340         err = reset(engine);
0341 
0342     /* Ensure the reset happens and kills the engine */
0343     if (err == 0)
0344         err = intel_selftest_wait_for_rq(rq);
0345 
0346     igt_spinner_end(&spin);
0347 
0348     if (err) {
0349         pr_err("%s reset failed\n", name);
0350         goto out_spin;
0351     }
0352 
0353     err = check_whitelist(ce);
0354     if (err) {
0355         pr_err("Whitelist not preserved in context across %s reset!\n",
0356                name);
0357         goto out_spin;
0358     }
0359 
0360     tmp = intel_context_create(engine);
0361     if (IS_ERR(tmp)) {
0362         err = PTR_ERR(tmp);
0363         goto out_spin;
0364     }
0365     intel_context_put(ce);
0366     ce = tmp;
0367 
0368     err = check_whitelist(ce);
0369     if (err) {
0370         pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
0371                name);
0372         goto out_spin;
0373     }
0374 
0375 out_spin:
0376     igt_spinner_fini(&spin);
0377 out_ctx:
0378     intel_context_put(ce);
0379     return err;
0380 }
0381 
0382 static struct i915_vma *create_batch(struct i915_address_space *vm)
0383 {
0384     struct drm_i915_gem_object *obj;
0385     struct i915_vma *vma;
0386     int err;
0387 
0388     obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
0389     if (IS_ERR(obj))
0390         return ERR_CAST(obj);
0391 
0392     vma = i915_vma_instance(obj, vm, NULL);
0393     if (IS_ERR(vma)) {
0394         err = PTR_ERR(vma);
0395         goto err_obj;
0396     }
0397 
0398     err = i915_vma_pin(vma, 0, 0, PIN_USER);
0399     if (err)
0400         goto err_obj;
0401 
0402     return vma;
0403 
0404 err_obj:
0405     i915_gem_object_put(obj);
0406     return ERR_PTR(err);
0407 }
0408 
0409 static u32 reg_write(u32 old, u32 new, u32 rsvd)
0410 {
0411     if (rsvd == 0x0000ffff) {
0412         old &= ~(new >> 16);
0413         old |= new & (new >> 16);
0414     } else {
0415         old &= ~rsvd;
0416         old |= new & rsvd;
0417     }
0418 
0419     return old;
0420 }
0421 
0422 static bool wo_register(struct intel_engine_cs *engine, u32 reg)
0423 {
0424     enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
0425     int i;
0426 
0427     if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
0428          RING_FORCE_TO_NONPRIV_ACCESS_WR)
0429         return true;
0430 
0431     for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
0432         if (wo_registers[i].platform == platform &&
0433             wo_registers[i].reg == reg)
0434             return true;
0435     }
0436 
0437     return false;
0438 }
0439 
0440 static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
0441 {
0442     reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
0443     switch (reg) {
0444     case 0x358:
0445     case 0x35c:
0446     case 0x3a8:
0447         return true;
0448 
0449     default:
0450         return false;
0451     }
0452 }
0453 
0454 static bool ro_register(u32 reg)
0455 {
0456     if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
0457          RING_FORCE_TO_NONPRIV_ACCESS_RD)
0458         return true;
0459 
0460     return false;
0461 }
0462 
0463 static int whitelist_writable_count(struct intel_engine_cs *engine)
0464 {
0465     int count = engine->whitelist.count;
0466     int i;
0467 
0468     for (i = 0; i < engine->whitelist.count; i++) {
0469         u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
0470 
0471         if (ro_register(reg))
0472             count--;
0473     }
0474 
0475     return count;
0476 }
0477 
0478 static int check_dirty_whitelist(struct intel_context *ce)
0479 {
0480     const u32 values[] = {
0481         0x00000000,
0482         0x01010101,
0483         0x10100101,
0484         0x03030303,
0485         0x30300303,
0486         0x05050505,
0487         0x50500505,
0488         0x0f0f0f0f,
0489         0xf00ff00f,
0490         0x10101010,
0491         0xf0f01010,
0492         0x30303030,
0493         0xa0a03030,
0494         0x50505050,
0495         0xc0c05050,
0496         0xf0f0f0f0,
0497         0x11111111,
0498         0x33333333,
0499         0x55555555,
0500         0x0000ffff,
0501         0x00ff00ff,
0502         0xff0000ff,
0503         0xffff00ff,
0504         0xffffffff,
0505     };
0506     struct intel_engine_cs *engine = ce->engine;
0507     struct i915_vma *scratch;
0508     struct i915_vma *batch;
0509     int err = 0, i, v, sz;
0510     u32 *cs, *results;
0511 
0512     sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
0513     scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
0514     if (IS_ERR(scratch))
0515         return PTR_ERR(scratch);
0516 
0517     batch = create_batch(ce->vm);
0518     if (IS_ERR(batch)) {
0519         err = PTR_ERR(batch);
0520         goto out_scratch;
0521     }
0522 
0523     for (i = 0; i < engine->whitelist.count; i++) {
0524         u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
0525         struct i915_gem_ww_ctx ww;
0526         u64 addr = scratch->node.start;
0527         struct i915_request *rq;
0528         u32 srm, lrm, rsvd;
0529         u32 expect;
0530         int idx;
0531         bool ro_reg;
0532 
0533         if (wo_register(engine, reg))
0534             continue;
0535 
0536         if (timestamp(engine, reg))
0537             continue; /* timestamps are expected to autoincrement */
0538 
0539         ro_reg = ro_register(reg);
0540 
0541         i915_gem_ww_ctx_init(&ww, false);
0542 retry:
0543         cs = NULL;
0544         err = i915_gem_object_lock(scratch->obj, &ww);
0545         if (!err)
0546             err = i915_gem_object_lock(batch->obj, &ww);
0547         if (!err)
0548             err = intel_context_pin_ww(ce, &ww);
0549         if (err)
0550             goto out;
0551 
0552         cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
0553         if (IS_ERR(cs)) {
0554             err = PTR_ERR(cs);
0555             goto out_ctx;
0556         }
0557 
0558         results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
0559         if (IS_ERR(results)) {
0560             err = PTR_ERR(results);
0561             goto out_unmap_batch;
0562         }
0563 
0564         /* Clear non priv flags */
0565         reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
0566 
0567         srm = MI_STORE_REGISTER_MEM;
0568         lrm = MI_LOAD_REGISTER_MEM;
0569         if (GRAPHICS_VER(engine->i915) >= 8)
0570             lrm++, srm++;
0571 
0572         pr_debug("%s: Writing garbage to %x\n",
0573              engine->name, reg);
0574 
0575         /* SRM original */
0576         *cs++ = srm;
0577         *cs++ = reg;
0578         *cs++ = lower_32_bits(addr);
0579         *cs++ = upper_32_bits(addr);
0580 
0581         idx = 1;
0582         for (v = 0; v < ARRAY_SIZE(values); v++) {
0583             /* LRI garbage */
0584             *cs++ = MI_LOAD_REGISTER_IMM(1);
0585             *cs++ = reg;
0586             *cs++ = values[v];
0587 
0588             /* SRM result */
0589             *cs++ = srm;
0590             *cs++ = reg;
0591             *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
0592             *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
0593             idx++;
0594         }
0595         for (v = 0; v < ARRAY_SIZE(values); v++) {
0596             /* LRI garbage */
0597             *cs++ = MI_LOAD_REGISTER_IMM(1);
0598             *cs++ = reg;
0599             *cs++ = ~values[v];
0600 
0601             /* SRM result */
0602             *cs++ = srm;
0603             *cs++ = reg;
0604             *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
0605             *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
0606             idx++;
0607         }
0608         GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
0609 
0610         /* LRM original -- don't leave garbage in the context! */
0611         *cs++ = lrm;
0612         *cs++ = reg;
0613         *cs++ = lower_32_bits(addr);
0614         *cs++ = upper_32_bits(addr);
0615 
0616         *cs++ = MI_BATCH_BUFFER_END;
0617 
0618         i915_gem_object_flush_map(batch->obj);
0619         i915_gem_object_unpin_map(batch->obj);
0620         intel_gt_chipset_flush(engine->gt);
0621         cs = NULL;
0622 
0623         rq = i915_request_create(ce);
0624         if (IS_ERR(rq)) {
0625             err = PTR_ERR(rq);
0626             goto out_unmap_scratch;
0627         }
0628 
0629         if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
0630             err = engine->emit_init_breadcrumb(rq);
0631             if (err)
0632                 goto err_request;
0633         }
0634 
0635         err = i915_request_await_object(rq, batch->obj, false);
0636         if (err == 0)
0637             err = i915_vma_move_to_active(batch, rq, 0);
0638         if (err)
0639             goto err_request;
0640 
0641         err = i915_request_await_object(rq, scratch->obj, true);
0642         if (err == 0)
0643             err = i915_vma_move_to_active(scratch, rq,
0644                               EXEC_OBJECT_WRITE);
0645         if (err)
0646             goto err_request;
0647 
0648         err = engine->emit_bb_start(rq,
0649                         batch->node.start, PAGE_SIZE,
0650                         0);
0651         if (err)
0652             goto err_request;
0653 
0654 err_request:
0655         err = request_add_sync(rq, err);
0656         if (err) {
0657             pr_err("%s: Futzing %x timedout; cancelling test\n",
0658                    engine->name, reg);
0659             intel_gt_set_wedged(engine->gt);
0660             goto out_unmap_scratch;
0661         }
0662 
0663         GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
0664         if (!ro_reg) {
0665             /* detect write masking */
0666             rsvd = results[ARRAY_SIZE(values)];
0667             if (!rsvd) {
0668                 pr_err("%s: Unable to write to whitelisted register %x\n",
0669                        engine->name, reg);
0670                 err = -EINVAL;
0671                 goto out_unmap_scratch;
0672             }
0673         } else {
0674             rsvd = 0;
0675         }
0676 
0677         expect = results[0];
0678         idx = 1;
0679         for (v = 0; v < ARRAY_SIZE(values); v++) {
0680             if (ro_reg)
0681                 expect = results[0];
0682             else
0683                 expect = reg_write(expect, values[v], rsvd);
0684 
0685             if (results[idx] != expect)
0686                 err++;
0687             idx++;
0688         }
0689         for (v = 0; v < ARRAY_SIZE(values); v++) {
0690             if (ro_reg)
0691                 expect = results[0];
0692             else
0693                 expect = reg_write(expect, ~values[v], rsvd);
0694 
0695             if (results[idx] != expect)
0696                 err++;
0697             idx++;
0698         }
0699         if (err) {
0700             pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
0701                    engine->name, err, reg);
0702 
0703             if (ro_reg)
0704                 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
0705                     engine->name, reg, results[0]);
0706             else
0707                 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
0708                     engine->name, reg, results[0], rsvd);
0709 
0710             expect = results[0];
0711             idx = 1;
0712             for (v = 0; v < ARRAY_SIZE(values); v++) {
0713                 u32 w = values[v];
0714 
0715                 if (ro_reg)
0716                     expect = results[0];
0717                 else
0718                     expect = reg_write(expect, w, rsvd);
0719                 pr_info("Wrote %08x, read %08x, expect %08x\n",
0720                     w, results[idx], expect);
0721                 idx++;
0722             }
0723             for (v = 0; v < ARRAY_SIZE(values); v++) {
0724                 u32 w = ~values[v];
0725 
0726                 if (ro_reg)
0727                     expect = results[0];
0728                 else
0729                     expect = reg_write(expect, w, rsvd);
0730                 pr_info("Wrote %08x, read %08x, expect %08x\n",
0731                     w, results[idx], expect);
0732                 idx++;
0733             }
0734 
0735             err = -EINVAL;
0736         }
0737 out_unmap_scratch:
0738         i915_gem_object_unpin_map(scratch->obj);
0739 out_unmap_batch:
0740         if (cs)
0741             i915_gem_object_unpin_map(batch->obj);
0742 out_ctx:
0743         intel_context_unpin(ce);
0744 out:
0745         if (err == -EDEADLK) {
0746             err = i915_gem_ww_ctx_backoff(&ww);
0747             if (!err)
0748                 goto retry;
0749         }
0750         i915_gem_ww_ctx_fini(&ww);
0751         if (err)
0752             break;
0753     }
0754 
0755     if (igt_flush_test(engine->i915))
0756         err = -EIO;
0757 
0758     i915_vma_unpin_and_release(&batch, 0);
0759 out_scratch:
0760     i915_vma_unpin_and_release(&scratch, 0);
0761     return err;
0762 }
0763 
0764 static int live_dirty_whitelist(void *arg)
0765 {
0766     struct intel_gt *gt = arg;
0767     struct intel_engine_cs *engine;
0768     enum intel_engine_id id;
0769 
0770     /* Can the user write to the whitelisted registers? */
0771 
0772     if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */
0773         return 0;
0774 
0775     for_each_engine(engine, gt, id) {
0776         struct intel_context *ce;
0777         int err;
0778 
0779         if (engine->whitelist.count == 0)
0780             continue;
0781 
0782         ce = intel_context_create(engine);
0783         if (IS_ERR(ce))
0784             return PTR_ERR(ce);
0785 
0786         err = check_dirty_whitelist(ce);
0787         intel_context_put(ce);
0788         if (err)
0789             return err;
0790     }
0791 
0792     return 0;
0793 }
0794 
0795 static int live_reset_whitelist(void *arg)
0796 {
0797     struct intel_gt *gt = arg;
0798     struct intel_engine_cs *engine;
0799     enum intel_engine_id id;
0800     int err = 0;
0801 
0802     /* If we reset the gpu, we should not lose the RING_NONPRIV */
0803     igt_global_reset_lock(gt);
0804 
0805     for_each_engine(engine, gt, id) {
0806         if (engine->whitelist.count == 0)
0807             continue;
0808 
0809         if (intel_has_reset_engine(gt)) {
0810             if (intel_engine_uses_guc(engine)) {
0811                 struct intel_selftest_saved_policy saved;
0812                 int err2;
0813 
0814                 err = intel_selftest_modify_policy(engine, &saved,
0815                                    SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
0816                 if (err)
0817                     goto out;
0818 
0819                 err = check_whitelist_across_reset(engine,
0820                                    do_guc_reset,
0821                                    "guc");
0822 
0823                 err2 = intel_selftest_restore_policy(engine, &saved);
0824                 if (err == 0)
0825                     err = err2;
0826             } else {
0827                 err = check_whitelist_across_reset(engine,
0828                                    do_engine_reset,
0829                                    "engine");
0830             }
0831 
0832             if (err)
0833                 goto out;
0834         }
0835 
0836         if (intel_has_gpu_reset(gt)) {
0837             err = check_whitelist_across_reset(engine,
0838                                do_device_reset,
0839                                "device");
0840             if (err)
0841                 goto out;
0842         }
0843     }
0844 
0845 out:
0846     igt_global_reset_unlock(gt);
0847     return err;
0848 }
0849 
0850 static int read_whitelisted_registers(struct intel_context *ce,
0851                       struct i915_vma *results)
0852 {
0853     struct intel_engine_cs *engine = ce->engine;
0854     struct i915_request *rq;
0855     int i, err = 0;
0856     u32 srm, *cs;
0857 
0858     rq = intel_context_create_request(ce);
0859     if (IS_ERR(rq))
0860         return PTR_ERR(rq);
0861 
0862     i915_vma_lock(results);
0863     err = i915_request_await_object(rq, results->obj, true);
0864     if (err == 0)
0865         err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
0866     i915_vma_unlock(results);
0867     if (err)
0868         goto err_req;
0869 
0870     srm = MI_STORE_REGISTER_MEM;
0871     if (GRAPHICS_VER(engine->i915) >= 8)
0872         srm++;
0873 
0874     cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
0875     if (IS_ERR(cs)) {
0876         err = PTR_ERR(cs);
0877         goto err_req;
0878     }
0879 
0880     for (i = 0; i < engine->whitelist.count; i++) {
0881         u64 offset = results->node.start + sizeof(u32) * i;
0882         u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
0883 
0884         /* Clear non priv flags */
0885         reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
0886 
0887         *cs++ = srm;
0888         *cs++ = reg;
0889         *cs++ = lower_32_bits(offset);
0890         *cs++ = upper_32_bits(offset);
0891     }
0892     intel_ring_advance(rq, cs);
0893 
0894 err_req:
0895     return request_add_sync(rq, err);
0896 }
0897 
0898 static int scrub_whitelisted_registers(struct intel_context *ce)
0899 {
0900     struct intel_engine_cs *engine = ce->engine;
0901     struct i915_request *rq;
0902     struct i915_vma *batch;
0903     int i, err = 0;
0904     u32 *cs;
0905 
0906     batch = create_batch(ce->vm);
0907     if (IS_ERR(batch))
0908         return PTR_ERR(batch);
0909 
0910     cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
0911     if (IS_ERR(cs)) {
0912         err = PTR_ERR(cs);
0913         goto err_batch;
0914     }
0915 
0916     *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
0917     for (i = 0; i < engine->whitelist.count; i++) {
0918         u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
0919 
0920         if (ro_register(reg))
0921             continue;
0922 
0923         /* Clear non priv flags */
0924         reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
0925 
0926         *cs++ = reg;
0927         *cs++ = 0xffffffff;
0928     }
0929     *cs++ = MI_BATCH_BUFFER_END;
0930 
0931     i915_gem_object_flush_map(batch->obj);
0932     intel_gt_chipset_flush(engine->gt);
0933 
0934     rq = intel_context_create_request(ce);
0935     if (IS_ERR(rq)) {
0936         err = PTR_ERR(rq);
0937         goto err_unpin;
0938     }
0939 
0940     if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
0941         err = engine->emit_init_breadcrumb(rq);
0942         if (err)
0943             goto err_request;
0944     }
0945 
0946     i915_vma_lock(batch);
0947     err = i915_request_await_object(rq, batch->obj, false);
0948     if (err == 0)
0949         err = i915_vma_move_to_active(batch, rq, 0);
0950     i915_vma_unlock(batch);
0951     if (err)
0952         goto err_request;
0953 
0954     /* Perform the writes from an unprivileged "user" batch */
0955     err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
0956 
0957 err_request:
0958     err = request_add_sync(rq, err);
0959 
0960 err_unpin:
0961     i915_gem_object_unpin_map(batch->obj);
0962 err_batch:
0963     i915_vma_unpin_and_release(&batch, 0);
0964     return err;
0965 }
0966 
0967 struct regmask {
0968     i915_reg_t reg;
0969     u8 graphics_ver;
0970 };
0971 
0972 static bool find_reg(struct drm_i915_private *i915,
0973              i915_reg_t reg,
0974              const struct regmask *tbl,
0975              unsigned long count)
0976 {
0977     u32 offset = i915_mmio_reg_offset(reg);
0978 
0979     while (count--) {
0980         if (GRAPHICS_VER(i915) == tbl->graphics_ver &&
0981             i915_mmio_reg_offset(tbl->reg) == offset)
0982             return true;
0983         tbl++;
0984     }
0985 
0986     return false;
0987 }
0988 
0989 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
0990 {
0991     /* Alas, we must pardon some whitelists. Mistakes already made */
0992     static const struct regmask pardon[] = {
0993         { GEN9_CTX_PREEMPT_REG, 9 },
0994         { GEN8_L3SQCREG4, 9 },
0995     };
0996 
0997     return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
0998 }
0999 
1000 static bool result_eq(struct intel_engine_cs *engine,
1001               u32 a, u32 b, i915_reg_t reg)
1002 {
1003     if (a != b && !pardon_reg(engine->i915, reg)) {
1004         pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
1005                i915_mmio_reg_offset(reg), a, b);
1006         return false;
1007     }
1008 
1009     return true;
1010 }
1011 
1012 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
1013 {
1014     /* Some registers do not seem to behave and our writes unreadable */
1015     static const struct regmask wo[] = {
1016         { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 },
1017     };
1018 
1019     return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
1020 }
1021 
1022 static bool result_neq(struct intel_engine_cs *engine,
1023                u32 a, u32 b, i915_reg_t reg)
1024 {
1025     if (a == b && !writeonly_reg(engine->i915, reg)) {
1026         pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
1027                i915_mmio_reg_offset(reg), a);
1028         return false;
1029     }
1030 
1031     return true;
1032 }
1033 
1034 static int
1035 check_whitelisted_registers(struct intel_engine_cs *engine,
1036                 struct i915_vma *A,
1037                 struct i915_vma *B,
1038                 bool (*fn)(struct intel_engine_cs *engine,
1039                        u32 a, u32 b,
1040                        i915_reg_t reg))
1041 {
1042     u32 *a, *b;
1043     int i, err;
1044 
1045     a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1046     if (IS_ERR(a))
1047         return PTR_ERR(a);
1048 
1049     b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1050     if (IS_ERR(b)) {
1051         err = PTR_ERR(b);
1052         goto err_a;
1053     }
1054 
1055     err = 0;
1056     for (i = 0; i < engine->whitelist.count; i++) {
1057         const struct i915_wa *wa = &engine->whitelist.list[i];
1058 
1059         if (i915_mmio_reg_offset(wa->reg) &
1060             RING_FORCE_TO_NONPRIV_ACCESS_RD)
1061             continue;
1062 
1063         if (!fn(engine, a[i], b[i], wa->reg))
1064             err = -EINVAL;
1065     }
1066 
1067     i915_gem_object_unpin_map(B->obj);
1068 err_a:
1069     i915_gem_object_unpin_map(A->obj);
1070     return err;
1071 }
1072 
1073 static int live_isolated_whitelist(void *arg)
1074 {
1075     struct intel_gt *gt = arg;
1076     struct {
1077         struct i915_vma *scratch[2];
1078     } client[2] = {};
1079     struct intel_engine_cs *engine;
1080     enum intel_engine_id id;
1081     int i, err = 0;
1082 
1083     /*
1084      * Check that a write into a whitelist register works, but
1085      * invisible to a second context.
1086      */
1087 
1088     if (!intel_engines_has_context_isolation(gt->i915))
1089         return 0;
1090 
1091     for (i = 0; i < ARRAY_SIZE(client); i++) {
1092         client[i].scratch[0] =
1093             __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1094         if (IS_ERR(client[i].scratch[0])) {
1095             err = PTR_ERR(client[i].scratch[0]);
1096             goto err;
1097         }
1098 
1099         client[i].scratch[1] =
1100             __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1101         if (IS_ERR(client[i].scratch[1])) {
1102             err = PTR_ERR(client[i].scratch[1]);
1103             i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1104             goto err;
1105         }
1106     }
1107 
1108     for_each_engine(engine, gt, id) {
1109         struct intel_context *ce[2];
1110 
1111         if (!engine->kernel_context->vm)
1112             continue;
1113 
1114         if (!whitelist_writable_count(engine))
1115             continue;
1116 
1117         ce[0] = intel_context_create(engine);
1118         if (IS_ERR(ce[0])) {
1119             err = PTR_ERR(ce[0]);
1120             break;
1121         }
1122         ce[1] = intel_context_create(engine);
1123         if (IS_ERR(ce[1])) {
1124             err = PTR_ERR(ce[1]);
1125             intel_context_put(ce[0]);
1126             break;
1127         }
1128 
1129         /* Read default values */
1130         err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1131         if (err)
1132             goto err_ce;
1133 
1134         /* Try to overwrite registers (should only affect ctx0) */
1135         err = scrub_whitelisted_registers(ce[0]);
1136         if (err)
1137             goto err_ce;
1138 
1139         /* Read values from ctx1, we expect these to be defaults */
1140         err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1141         if (err)
1142             goto err_ce;
1143 
1144         /* Verify that both reads return the same default values */
1145         err = check_whitelisted_registers(engine,
1146                           client[0].scratch[0],
1147                           client[1].scratch[0],
1148                           result_eq);
1149         if (err)
1150             goto err_ce;
1151 
1152         /* Read back the updated values in ctx0 */
1153         err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1154         if (err)
1155             goto err_ce;
1156 
1157         /* User should be granted privilege to overwhite regs */
1158         err = check_whitelisted_registers(engine,
1159                           client[0].scratch[0],
1160                           client[0].scratch[1],
1161                           result_neq);
1162 err_ce:
1163         intel_context_put(ce[1]);
1164         intel_context_put(ce[0]);
1165         if (err)
1166             break;
1167     }
1168 
1169 err:
1170     for (i = 0; i < ARRAY_SIZE(client); i++) {
1171         i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1172         i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1173     }
1174 
1175     if (igt_flush_test(gt->i915))
1176         err = -EIO;
1177 
1178     return err;
1179 }
1180 
1181 static bool
1182 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1183         const char *str)
1184 {
1185     struct intel_engine_cs *engine;
1186     enum intel_engine_id id;
1187     bool ok = true;
1188 
1189     ok &= wa_list_verify(gt, &lists->gt_wa_list, str);
1190 
1191     for_each_engine(engine, gt, id) {
1192         struct intel_context *ce;
1193 
1194         ce = intel_context_create(engine);
1195         if (IS_ERR(ce))
1196             return false;
1197 
1198         ok &= engine_wa_list_verify(ce,
1199                         &lists->engine[id].wa_list,
1200                         str) == 0;
1201 
1202         ok &= engine_wa_list_verify(ce,
1203                         &lists->engine[id].ctx_wa_list,
1204                         str) == 0;
1205 
1206         intel_context_put(ce);
1207     }
1208 
1209     return ok;
1210 }
1211 
1212 static int
1213 live_gpu_reset_workarounds(void *arg)
1214 {
1215     struct intel_gt *gt = arg;
1216     intel_wakeref_t wakeref;
1217     struct wa_lists *lists;
1218     bool ok;
1219 
1220     if (!intel_has_gpu_reset(gt))
1221         return 0;
1222 
1223     lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1224     if (!lists)
1225         return -ENOMEM;
1226 
1227     pr_info("Verifying after GPU reset...\n");
1228 
1229     igt_global_reset_lock(gt);
1230     wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1231 
1232     reference_lists_init(gt, lists);
1233 
1234     ok = verify_wa_lists(gt, lists, "before reset");
1235     if (!ok)
1236         goto out;
1237 
1238     intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1239 
1240     ok = verify_wa_lists(gt, lists, "after reset");
1241 
1242 out:
1243     reference_lists_fini(gt, lists);
1244     intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1245     igt_global_reset_unlock(gt);
1246     kfree(lists);
1247 
1248     return ok ? 0 : -ESRCH;
1249 }
1250 
1251 static int
1252 live_engine_reset_workarounds(void *arg)
1253 {
1254     struct intel_gt *gt = arg;
1255     struct intel_engine_cs *engine;
1256     enum intel_engine_id id;
1257     struct intel_context *ce;
1258     struct igt_spinner spin;
1259     struct i915_request *rq;
1260     intel_wakeref_t wakeref;
1261     struct wa_lists *lists;
1262     int ret = 0;
1263 
1264     if (!intel_has_reset_engine(gt))
1265         return 0;
1266 
1267     lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1268     if (!lists)
1269         return -ENOMEM;
1270 
1271     igt_global_reset_lock(gt);
1272     wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1273 
1274     reference_lists_init(gt, lists);
1275 
1276     for_each_engine(engine, gt, id) {
1277         struct intel_selftest_saved_policy saved;
1278         bool using_guc = intel_engine_uses_guc(engine);
1279         bool ok;
1280         int ret2;
1281 
1282         pr_info("Verifying after %s reset...\n", engine->name);
1283         ret = intel_selftest_modify_policy(engine, &saved,
1284                            SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
1285         if (ret)
1286             break;
1287 
1288         ce = intel_context_create(engine);
1289         if (IS_ERR(ce)) {
1290             ret = PTR_ERR(ce);
1291             goto restore;
1292         }
1293 
1294         if (!using_guc) {
1295             ok = verify_wa_lists(gt, lists, "before reset");
1296             if (!ok) {
1297                 ret = -ESRCH;
1298                 goto err;
1299             }
1300 
1301             ret = intel_engine_reset(engine, "live_workarounds:idle");
1302             if (ret) {
1303                 pr_err("%s: Reset failed while idle\n", engine->name);
1304                 goto err;
1305             }
1306 
1307             ok = verify_wa_lists(gt, lists, "after idle reset");
1308             if (!ok) {
1309                 ret = -ESRCH;
1310                 goto err;
1311             }
1312         }
1313 
1314         ret = igt_spinner_init(&spin, engine->gt);
1315         if (ret)
1316             goto err;
1317 
1318         rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1319         if (IS_ERR(rq)) {
1320             ret = PTR_ERR(rq);
1321             igt_spinner_fini(&spin);
1322             goto err;
1323         }
1324 
1325         ret = request_add_spin(rq, &spin);
1326         if (ret) {
1327             pr_err("%s: Spinner failed to start\n", engine->name);
1328             igt_spinner_fini(&spin);
1329             goto err;
1330         }
1331 
1332         /* Ensure the spinner hasn't aborted */
1333         if (i915_request_completed(rq)) {
1334             ret = -ETIMEDOUT;
1335             goto skip;
1336         }
1337 
1338         if (!using_guc) {
1339             ret = intel_engine_reset(engine, "live_workarounds:active");
1340             if (ret) {
1341                 pr_err("%s: Reset failed on an active spinner\n",
1342                        engine->name);
1343                 igt_spinner_fini(&spin);
1344                 goto err;
1345             }
1346         }
1347 
1348         /* Ensure the reset happens and kills the engine */
1349         if (ret == 0)
1350             ret = intel_selftest_wait_for_rq(rq);
1351 
1352 skip:
1353         igt_spinner_end(&spin);
1354         igt_spinner_fini(&spin);
1355 
1356         ok = verify_wa_lists(gt, lists, "after busy reset");
1357         if (!ok)
1358             ret = -ESRCH;
1359 
1360 err:
1361         intel_context_put(ce);
1362 
1363 restore:
1364         ret2 = intel_selftest_restore_policy(engine, &saved);
1365         if (ret == 0)
1366             ret = ret2;
1367         if (ret)
1368             break;
1369     }
1370 
1371     reference_lists_fini(gt, lists);
1372     intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1373     igt_global_reset_unlock(gt);
1374     kfree(lists);
1375 
1376     igt_flush_test(gt->i915);
1377 
1378     return ret;
1379 }
1380 
1381 int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1382 {
1383     static const struct i915_subtest tests[] = {
1384         SUBTEST(live_dirty_whitelist),
1385         SUBTEST(live_reset_whitelist),
1386         SUBTEST(live_isolated_whitelist),
1387         SUBTEST(live_gpu_reset_workarounds),
1388         SUBTEST(live_engine_reset_workarounds),
1389     };
1390 
1391     if (intel_gt_is_wedged(to_gt(i915)))
1392         return 0;
1393 
1394     return intel_gt_live_subtests(tests, to_gt(i915));
1395 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!