Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2018 Intel Corporation
0004  */
0005 
0006 #include <linux/prime_numbers.h>
0007 
0008 #include "gem/i915_gem_internal.h"
0009 
0010 #include "i915_selftest.h"
0011 #include "intel_engine_heartbeat.h"
0012 #include "intel_engine_pm.h"
0013 #include "intel_reset.h"
0014 #include "intel_ring.h"
0015 #include "selftest_engine_heartbeat.h"
0016 #include "selftests/i915_random.h"
0017 #include "selftests/igt_flush_test.h"
0018 #include "selftests/igt_live_test.h"
0019 #include "selftests/igt_spinner.h"
0020 #include "selftests/lib_sw_fence.h"
0021 #include "shmem_utils.h"
0022 
0023 #include "gem/selftests/igt_gem_utils.h"
0024 #include "gem/selftests/mock_context.h"
0025 
0026 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
0027 #define NUM_GPR 16
0028 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
0029 
0030 static struct i915_vma *create_scratch(struct intel_gt *gt)
0031 {
0032     return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
0033 }
0034 
0035 static bool is_active(struct i915_request *rq)
0036 {
0037     if (i915_request_is_active(rq))
0038         return true;
0039 
0040     if (i915_request_on_hold(rq))
0041         return true;
0042 
0043     if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
0044         return true;
0045 
0046     return false;
0047 }
0048 
0049 static int wait_for_submit(struct intel_engine_cs *engine,
0050                struct i915_request *rq,
0051                unsigned long timeout)
0052 {
0053     /* Ignore our own attempts to suppress excess tasklets */
0054     tasklet_hi_schedule(&engine->sched_engine->tasklet);
0055 
0056     timeout += jiffies;
0057     do {
0058         bool done = time_after(jiffies, timeout);
0059 
0060         if (i915_request_completed(rq)) /* that was quick! */
0061             return 0;
0062 
0063         /* Wait until the HW has acknowleged the submission (or err) */
0064         intel_engine_flush_submission(engine);
0065         if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
0066             return 0;
0067 
0068         if (done)
0069             return -ETIME;
0070 
0071         cond_resched();
0072     } while (1);
0073 }
0074 
0075 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
0076 {
0077     const u32 offset =
0078         i915_ggtt_offset(ce->engine->status_page.vma) +
0079         offset_in_page(slot);
0080     struct i915_request *rq;
0081     u32 *cs;
0082 
0083     rq = intel_context_create_request(ce);
0084     if (IS_ERR(rq))
0085         return PTR_ERR(rq);
0086 
0087     cs = intel_ring_begin(rq, 4);
0088     if (IS_ERR(cs)) {
0089         i915_request_add(rq);
0090         return PTR_ERR(cs);
0091     }
0092 
0093     *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
0094     *cs++ = offset;
0095     *cs++ = 0;
0096     *cs++ = 1;
0097 
0098     intel_ring_advance(rq, cs);
0099 
0100     rq->sched.attr.priority = I915_PRIORITY_BARRIER;
0101     i915_request_add(rq);
0102     return 0;
0103 }
0104 
0105 static int context_flush(struct intel_context *ce, long timeout)
0106 {
0107     struct i915_request *rq;
0108     struct dma_fence *fence;
0109     int err = 0;
0110 
0111     rq = intel_engine_create_kernel_request(ce->engine);
0112     if (IS_ERR(rq))
0113         return PTR_ERR(rq);
0114 
0115     fence = i915_active_fence_get(&ce->timeline->last_request);
0116     if (fence) {
0117         i915_request_await_dma_fence(rq, fence);
0118         dma_fence_put(fence);
0119     }
0120 
0121     rq = i915_request_get(rq);
0122     i915_request_add(rq);
0123     if (i915_request_wait(rq, 0, timeout) < 0)
0124         err = -ETIME;
0125     i915_request_put(rq);
0126 
0127     rmb(); /* We know the request is written, make sure all state is too! */
0128     return err;
0129 }
0130 
0131 static int get_lri_mask(struct intel_engine_cs *engine, u32 lri)
0132 {
0133     if ((lri & MI_LRI_LRM_CS_MMIO) == 0)
0134         return ~0u;
0135 
0136     if (GRAPHICS_VER(engine->i915) < 12)
0137         return 0xfff;
0138 
0139     switch (engine->class) {
0140     default:
0141     case RENDER_CLASS:
0142     case COMPUTE_CLASS:
0143         return 0x07ff;
0144     case COPY_ENGINE_CLASS:
0145         return 0x0fff;
0146     case VIDEO_DECODE_CLASS:
0147     case VIDEO_ENHANCEMENT_CLASS:
0148         return 0x3fff;
0149     }
0150 }
0151 
0152 static int live_lrc_layout(void *arg)
0153 {
0154     struct intel_gt *gt = arg;
0155     struct intel_engine_cs *engine;
0156     enum intel_engine_id id;
0157     u32 *lrc;
0158     int err;
0159 
0160     /*
0161      * Check the registers offsets we use to create the initial reg state
0162      * match the layout saved by HW.
0163      */
0164 
0165     lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
0166     if (!lrc)
0167         return -ENOMEM;
0168     GEM_BUG_ON(offset_in_page(lrc));
0169 
0170     err = 0;
0171     for_each_engine(engine, gt, id) {
0172         u32 *hw;
0173         int dw;
0174 
0175         if (!engine->default_state)
0176             continue;
0177 
0178         hw = shmem_pin_map(engine->default_state);
0179         if (!hw) {
0180             err = -ENOMEM;
0181             break;
0182         }
0183         hw += LRC_STATE_OFFSET / sizeof(*hw);
0184 
0185         __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
0186                 engine->kernel_context, engine, true);
0187 
0188         dw = 0;
0189         do {
0190             u32 lri = READ_ONCE(hw[dw]);
0191             u32 lri_mask;
0192 
0193             if (lri == 0) {
0194                 dw++;
0195                 continue;
0196             }
0197 
0198             if (lrc[dw] == 0) {
0199                 pr_debug("%s: skipped instruction %x at dword %d\n",
0200                      engine->name, lri, dw);
0201                 dw++;
0202                 continue;
0203             }
0204 
0205             if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
0206                 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
0207                        engine->name, dw, lri);
0208                 err = -EINVAL;
0209                 break;
0210             }
0211 
0212             if (lrc[dw] != lri) {
0213                 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
0214                        engine->name, dw, lri, lrc[dw]);
0215                 err = -EINVAL;
0216                 break;
0217             }
0218 
0219             /*
0220              * When bit 19 of MI_LOAD_REGISTER_IMM instruction
0221              * opcode is set on Gen12+ devices, HW does not
0222              * care about certain register address offsets, and
0223              * instead check the following for valid address
0224              * ranges on specific engines:
0225              * RCS && CCS: BITS(0 - 10)
0226              * BCS: BITS(0 - 11)
0227              * VECS && VCS: BITS(0 - 13)
0228              */
0229             lri_mask = get_lri_mask(engine, lri);
0230 
0231             lri &= 0x7f;
0232             lri++;
0233             dw++;
0234 
0235             while (lri) {
0236                 u32 offset = READ_ONCE(hw[dw]);
0237 
0238                 if ((offset ^ lrc[dw]) & lri_mask) {
0239                     pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
0240                            engine->name, dw, offset, lrc[dw]);
0241                     err = -EINVAL;
0242                     break;
0243                 }
0244 
0245                 /*
0246                  * Skip over the actual register value as we
0247                  * expect that to differ.
0248                  */
0249                 dw += 2;
0250                 lri -= 2;
0251             }
0252         } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
0253 
0254         if (err) {
0255             pr_info("%s: HW register image:\n", engine->name);
0256             igt_hexdump(hw, PAGE_SIZE);
0257 
0258             pr_info("%s: SW register image:\n", engine->name);
0259             igt_hexdump(lrc, PAGE_SIZE);
0260         }
0261 
0262         shmem_unpin_map(engine->default_state, hw);
0263         if (err)
0264             break;
0265     }
0266 
0267     free_page((unsigned long)lrc);
0268     return err;
0269 }
0270 
0271 static int find_offset(const u32 *lri, u32 offset)
0272 {
0273     int i;
0274 
0275     for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
0276         if (lri[i] == offset)
0277             return i;
0278 
0279     return -1;
0280 }
0281 
0282 static int live_lrc_fixed(void *arg)
0283 {
0284     struct intel_gt *gt = arg;
0285     struct intel_engine_cs *engine;
0286     enum intel_engine_id id;
0287     int err = 0;
0288 
0289     /*
0290      * Check the assumed register offsets match the actual locations in
0291      * the context image.
0292      */
0293 
0294     for_each_engine(engine, gt, id) {
0295         const struct {
0296             u32 reg;
0297             u32 offset;
0298             const char *name;
0299         } tbl[] = {
0300             {
0301                 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
0302                 CTX_RING_START - 1,
0303                 "RING_START"
0304             },
0305             {
0306                 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
0307                 CTX_RING_CTL - 1,
0308                 "RING_CTL"
0309             },
0310             {
0311                 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
0312                 CTX_RING_HEAD - 1,
0313                 "RING_HEAD"
0314             },
0315             {
0316                 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
0317                 CTX_RING_TAIL - 1,
0318                 "RING_TAIL"
0319             },
0320             {
0321                 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
0322                 lrc_ring_mi_mode(engine),
0323                 "RING_MI_MODE"
0324             },
0325             {
0326                 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
0327                 CTX_BB_STATE - 1,
0328                 "BB_STATE"
0329             },
0330             {
0331                 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
0332                 lrc_ring_wa_bb_per_ctx(engine),
0333                 "RING_BB_PER_CTX_PTR"
0334             },
0335             {
0336                 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
0337                 lrc_ring_indirect_ptr(engine),
0338                 "RING_INDIRECT_CTX_PTR"
0339             },
0340             {
0341                 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
0342                 lrc_ring_indirect_offset(engine),
0343                 "RING_INDIRECT_CTX_OFFSET"
0344             },
0345             {
0346                 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
0347                 CTX_TIMESTAMP - 1,
0348                 "RING_CTX_TIMESTAMP"
0349             },
0350             {
0351                 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
0352                 lrc_ring_gpr0(engine),
0353                 "RING_CS_GPR0"
0354             },
0355             {
0356                 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
0357                 lrc_ring_cmd_buf_cctl(engine),
0358                 "RING_CMD_BUF_CCTL"
0359             },
0360             { },
0361         }, *t;
0362         u32 *hw;
0363 
0364         if (!engine->default_state)
0365             continue;
0366 
0367         hw = shmem_pin_map(engine->default_state);
0368         if (!hw) {
0369             err = -ENOMEM;
0370             break;
0371         }
0372         hw += LRC_STATE_OFFSET / sizeof(*hw);
0373 
0374         for (t = tbl; t->name; t++) {
0375             int dw = find_offset(hw, t->reg);
0376 
0377             if (dw != t->offset) {
0378                 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
0379                        engine->name,
0380                        t->name,
0381                        t->reg,
0382                        dw,
0383                        t->offset);
0384                 err = -EINVAL;
0385             }
0386         }
0387 
0388         shmem_unpin_map(engine->default_state, hw);
0389     }
0390 
0391     return err;
0392 }
0393 
0394 static int __live_lrc_state(struct intel_engine_cs *engine,
0395                 struct i915_vma *scratch)
0396 {
0397     struct intel_context *ce;
0398     struct i915_request *rq;
0399     struct i915_gem_ww_ctx ww;
0400     enum {
0401         RING_START_IDX = 0,
0402         RING_TAIL_IDX,
0403         MAX_IDX
0404     };
0405     u32 expected[MAX_IDX];
0406     u32 *cs;
0407     int err;
0408     int n;
0409 
0410     ce = intel_context_create(engine);
0411     if (IS_ERR(ce))
0412         return PTR_ERR(ce);
0413 
0414     i915_gem_ww_ctx_init(&ww, false);
0415 retry:
0416     err = i915_gem_object_lock(scratch->obj, &ww);
0417     if (!err)
0418         err = intel_context_pin_ww(ce, &ww);
0419     if (err)
0420         goto err_put;
0421 
0422     rq = i915_request_create(ce);
0423     if (IS_ERR(rq)) {
0424         err = PTR_ERR(rq);
0425         goto err_unpin;
0426     }
0427 
0428     cs = intel_ring_begin(rq, 4 * MAX_IDX);
0429     if (IS_ERR(cs)) {
0430         err = PTR_ERR(cs);
0431         i915_request_add(rq);
0432         goto err_unpin;
0433     }
0434 
0435     *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
0436     *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
0437     *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
0438     *cs++ = 0;
0439 
0440     expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
0441 
0442     *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
0443     *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
0444     *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
0445     *cs++ = 0;
0446 
0447     err = i915_request_await_object(rq, scratch->obj, true);
0448     if (!err)
0449         err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
0450 
0451     i915_request_get(rq);
0452     i915_request_add(rq);
0453     if (err)
0454         goto err_rq;
0455 
0456     intel_engine_flush_submission(engine);
0457     expected[RING_TAIL_IDX] = ce->ring->tail;
0458 
0459     if (i915_request_wait(rq, 0, HZ / 5) < 0) {
0460         err = -ETIME;
0461         goto err_rq;
0462     }
0463 
0464     cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
0465     if (IS_ERR(cs)) {
0466         err = PTR_ERR(cs);
0467         goto err_rq;
0468     }
0469 
0470     for (n = 0; n < MAX_IDX; n++) {
0471         if (cs[n] != expected[n]) {
0472             pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
0473                    engine->name, n, cs[n], expected[n]);
0474             err = -EINVAL;
0475             break;
0476         }
0477     }
0478 
0479     i915_gem_object_unpin_map(scratch->obj);
0480 
0481 err_rq:
0482     i915_request_put(rq);
0483 err_unpin:
0484     intel_context_unpin(ce);
0485 err_put:
0486     if (err == -EDEADLK) {
0487         err = i915_gem_ww_ctx_backoff(&ww);
0488         if (!err)
0489             goto retry;
0490     }
0491     i915_gem_ww_ctx_fini(&ww);
0492     intel_context_put(ce);
0493     return err;
0494 }
0495 
0496 static int live_lrc_state(void *arg)
0497 {
0498     struct intel_gt *gt = arg;
0499     struct intel_engine_cs *engine;
0500     struct i915_vma *scratch;
0501     enum intel_engine_id id;
0502     int err = 0;
0503 
0504     /*
0505      * Check the live register state matches what we expect for this
0506      * intel_context.
0507      */
0508 
0509     scratch = create_scratch(gt);
0510     if (IS_ERR(scratch))
0511         return PTR_ERR(scratch);
0512 
0513     for_each_engine(engine, gt, id) {
0514         err = __live_lrc_state(engine, scratch);
0515         if (err)
0516             break;
0517     }
0518 
0519     if (igt_flush_test(gt->i915))
0520         err = -EIO;
0521 
0522     i915_vma_unpin_and_release(&scratch, 0);
0523     return err;
0524 }
0525 
0526 static int gpr_make_dirty(struct intel_context *ce)
0527 {
0528     struct i915_request *rq;
0529     u32 *cs;
0530     int n;
0531 
0532     rq = intel_context_create_request(ce);
0533     if (IS_ERR(rq))
0534         return PTR_ERR(rq);
0535 
0536     cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
0537     if (IS_ERR(cs)) {
0538         i915_request_add(rq);
0539         return PTR_ERR(cs);
0540     }
0541 
0542     *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
0543     for (n = 0; n < NUM_GPR_DW; n++) {
0544         *cs++ = CS_GPR(ce->engine, n);
0545         *cs++ = STACK_MAGIC;
0546     }
0547     *cs++ = MI_NOOP;
0548 
0549     intel_ring_advance(rq, cs);
0550 
0551     rq->sched.attr.priority = I915_PRIORITY_BARRIER;
0552     i915_request_add(rq);
0553 
0554     return 0;
0555 }
0556 
0557 static struct i915_request *
0558 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
0559 {
0560     const u32 offset =
0561         i915_ggtt_offset(ce->engine->status_page.vma) +
0562         offset_in_page(slot);
0563     struct i915_request *rq;
0564     u32 *cs;
0565     int err;
0566     int n;
0567 
0568     rq = intel_context_create_request(ce);
0569     if (IS_ERR(rq))
0570         return rq;
0571 
0572     cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
0573     if (IS_ERR(cs)) {
0574         i915_request_add(rq);
0575         return ERR_CAST(cs);
0576     }
0577 
0578     *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
0579     *cs++ = MI_NOOP;
0580 
0581     *cs++ = MI_SEMAPHORE_WAIT |
0582         MI_SEMAPHORE_GLOBAL_GTT |
0583         MI_SEMAPHORE_POLL |
0584         MI_SEMAPHORE_SAD_NEQ_SDD;
0585     *cs++ = 0;
0586     *cs++ = offset;
0587     *cs++ = 0;
0588 
0589     for (n = 0; n < NUM_GPR_DW; n++) {
0590         *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
0591         *cs++ = CS_GPR(ce->engine, n);
0592         *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
0593         *cs++ = 0;
0594     }
0595 
0596     i915_vma_lock(scratch);
0597     err = i915_request_await_object(rq, scratch->obj, true);
0598     if (!err)
0599         err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
0600     i915_vma_unlock(scratch);
0601 
0602     i915_request_get(rq);
0603     i915_request_add(rq);
0604     if (err) {
0605         i915_request_put(rq);
0606         rq = ERR_PTR(err);
0607     }
0608 
0609     return rq;
0610 }
0611 
0612 static int __live_lrc_gpr(struct intel_engine_cs *engine,
0613               struct i915_vma *scratch,
0614               bool preempt)
0615 {
0616     u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
0617     struct intel_context *ce;
0618     struct i915_request *rq;
0619     u32 *cs;
0620     int err;
0621     int n;
0622 
0623     if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)
0624         return 0; /* GPR only on rcs0 for gen8 */
0625 
0626     err = gpr_make_dirty(engine->kernel_context);
0627     if (err)
0628         return err;
0629 
0630     ce = intel_context_create(engine);
0631     if (IS_ERR(ce))
0632         return PTR_ERR(ce);
0633 
0634     rq = __gpr_read(ce, scratch, slot);
0635     if (IS_ERR(rq)) {
0636         err = PTR_ERR(rq);
0637         goto err_put;
0638     }
0639 
0640     err = wait_for_submit(engine, rq, HZ / 2);
0641     if (err)
0642         goto err_rq;
0643 
0644     if (preempt) {
0645         err = gpr_make_dirty(engine->kernel_context);
0646         if (err)
0647             goto err_rq;
0648 
0649         err = emit_semaphore_signal(engine->kernel_context, slot);
0650         if (err)
0651             goto err_rq;
0652 
0653         err = wait_for_submit(engine, rq, HZ / 2);
0654         if (err)
0655             goto err_rq;
0656     } else {
0657         slot[0] = 1;
0658         wmb();
0659     }
0660 
0661     if (i915_request_wait(rq, 0, HZ / 5) < 0) {
0662         err = -ETIME;
0663         goto err_rq;
0664     }
0665 
0666     cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
0667     if (IS_ERR(cs)) {
0668         err = PTR_ERR(cs);
0669         goto err_rq;
0670     }
0671 
0672     for (n = 0; n < NUM_GPR_DW; n++) {
0673         if (cs[n]) {
0674             pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
0675                    engine->name,
0676                    n / 2, n & 1 ? "udw" : "ldw",
0677                    cs[n]);
0678             err = -EINVAL;
0679             break;
0680         }
0681     }
0682 
0683     i915_gem_object_unpin_map(scratch->obj);
0684 
0685 err_rq:
0686     memset32(&slot[0], -1, 4);
0687     wmb();
0688     i915_request_put(rq);
0689 err_put:
0690     intel_context_put(ce);
0691     return err;
0692 }
0693 
0694 static int live_lrc_gpr(void *arg)
0695 {
0696     struct intel_gt *gt = arg;
0697     struct intel_engine_cs *engine;
0698     struct i915_vma *scratch;
0699     enum intel_engine_id id;
0700     int err = 0;
0701 
0702     /*
0703      * Check that GPR registers are cleared in new contexts as we need
0704      * to avoid leaking any information from previous contexts.
0705      */
0706 
0707     scratch = create_scratch(gt);
0708     if (IS_ERR(scratch))
0709         return PTR_ERR(scratch);
0710 
0711     for_each_engine(engine, gt, id) {
0712         st_engine_heartbeat_disable(engine);
0713 
0714         err = __live_lrc_gpr(engine, scratch, false);
0715         if (err)
0716             goto err;
0717 
0718         err = __live_lrc_gpr(engine, scratch, true);
0719         if (err)
0720             goto err;
0721 
0722 err:
0723         st_engine_heartbeat_enable(engine);
0724         if (igt_flush_test(gt->i915))
0725             err = -EIO;
0726         if (err)
0727             break;
0728     }
0729 
0730     i915_vma_unpin_and_release(&scratch, 0);
0731     return err;
0732 }
0733 
0734 static struct i915_request *
0735 create_timestamp(struct intel_context *ce, void *slot, int idx)
0736 {
0737     const u32 offset =
0738         i915_ggtt_offset(ce->engine->status_page.vma) +
0739         offset_in_page(slot);
0740     struct i915_request *rq;
0741     u32 *cs;
0742     int err;
0743 
0744     rq = intel_context_create_request(ce);
0745     if (IS_ERR(rq))
0746         return rq;
0747 
0748     cs = intel_ring_begin(rq, 10);
0749     if (IS_ERR(cs)) {
0750         err = PTR_ERR(cs);
0751         goto err;
0752     }
0753 
0754     *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
0755     *cs++ = MI_NOOP;
0756 
0757     *cs++ = MI_SEMAPHORE_WAIT |
0758         MI_SEMAPHORE_GLOBAL_GTT |
0759         MI_SEMAPHORE_POLL |
0760         MI_SEMAPHORE_SAD_NEQ_SDD;
0761     *cs++ = 0;
0762     *cs++ = offset;
0763     *cs++ = 0;
0764 
0765     *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
0766     *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
0767     *cs++ = offset + idx * sizeof(u32);
0768     *cs++ = 0;
0769 
0770     intel_ring_advance(rq, cs);
0771 
0772     err = 0;
0773 err:
0774     i915_request_get(rq);
0775     i915_request_add(rq);
0776     if (err) {
0777         i915_request_put(rq);
0778         return ERR_PTR(err);
0779     }
0780 
0781     return rq;
0782 }
0783 
0784 struct lrc_timestamp {
0785     struct intel_engine_cs *engine;
0786     struct intel_context *ce[2];
0787     u32 poison;
0788 };
0789 
0790 static bool timestamp_advanced(u32 start, u32 end)
0791 {
0792     return (s32)(end - start) > 0;
0793 }
0794 
0795 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
0796 {
0797     u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
0798     struct i915_request *rq;
0799     u32 timestamp;
0800     int err = 0;
0801 
0802     arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
0803     rq = create_timestamp(arg->ce[0], slot, 1);
0804     if (IS_ERR(rq))
0805         return PTR_ERR(rq);
0806 
0807     err = wait_for_submit(rq->engine, rq, HZ / 2);
0808     if (err)
0809         goto err;
0810 
0811     if (preempt) {
0812         arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
0813         err = emit_semaphore_signal(arg->ce[1], slot);
0814         if (err)
0815             goto err;
0816     } else {
0817         slot[0] = 1;
0818         wmb();
0819     }
0820 
0821     /* And wait for switch to kernel (to save our context to memory) */
0822     err = context_flush(arg->ce[0], HZ / 2);
0823     if (err)
0824         goto err;
0825 
0826     if (!timestamp_advanced(arg->poison, slot[1])) {
0827         pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
0828                arg->engine->name, preempt ? "preempt" : "simple",
0829                arg->poison, slot[1]);
0830         err = -EINVAL;
0831     }
0832 
0833     timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
0834     if (!timestamp_advanced(slot[1], timestamp)) {
0835         pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
0836                arg->engine->name, preempt ? "preempt" : "simple",
0837                slot[1], timestamp);
0838         err = -EINVAL;
0839     }
0840 
0841 err:
0842     memset32(slot, -1, 4);
0843     i915_request_put(rq);
0844     return err;
0845 }
0846 
0847 static int live_lrc_timestamp(void *arg)
0848 {
0849     struct lrc_timestamp data = {};
0850     struct intel_gt *gt = arg;
0851     enum intel_engine_id id;
0852     const u32 poison[] = {
0853         0,
0854         S32_MAX,
0855         (u32)S32_MAX + 1,
0856         U32_MAX,
0857     };
0858 
0859     /*
0860      * We want to verify that the timestamp is saved and restore across
0861      * context switches and is monotonic.
0862      *
0863      * So we do this with a little bit of LRC poisoning to check various
0864      * boundary conditions, and see what happens if we preempt the context
0865      * with a second request (carrying more poison into the timestamp).
0866      */
0867 
0868     for_each_engine(data.engine, gt, id) {
0869         int i, err = 0;
0870 
0871         st_engine_heartbeat_disable(data.engine);
0872 
0873         for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
0874             struct intel_context *tmp;
0875 
0876             tmp = intel_context_create(data.engine);
0877             if (IS_ERR(tmp)) {
0878                 err = PTR_ERR(tmp);
0879                 goto err;
0880             }
0881 
0882             err = intel_context_pin(tmp);
0883             if (err) {
0884                 intel_context_put(tmp);
0885                 goto err;
0886             }
0887 
0888             data.ce[i] = tmp;
0889         }
0890 
0891         for (i = 0; i < ARRAY_SIZE(poison); i++) {
0892             data.poison = poison[i];
0893 
0894             err = __lrc_timestamp(&data, false);
0895             if (err)
0896                 break;
0897 
0898             err = __lrc_timestamp(&data, true);
0899             if (err)
0900                 break;
0901         }
0902 
0903 err:
0904         st_engine_heartbeat_enable(data.engine);
0905         for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
0906             if (!data.ce[i])
0907                 break;
0908 
0909             intel_context_unpin(data.ce[i]);
0910             intel_context_put(data.ce[i]);
0911         }
0912 
0913         if (igt_flush_test(gt->i915))
0914             err = -EIO;
0915         if (err)
0916             return err;
0917     }
0918 
0919     return 0;
0920 }
0921 
0922 static struct i915_vma *
0923 create_user_vma(struct i915_address_space *vm, unsigned long size)
0924 {
0925     struct drm_i915_gem_object *obj;
0926     struct i915_vma *vma;
0927     int err;
0928 
0929     obj = i915_gem_object_create_internal(vm->i915, size);
0930     if (IS_ERR(obj))
0931         return ERR_CAST(obj);
0932 
0933     vma = i915_vma_instance(obj, vm, NULL);
0934     if (IS_ERR(vma)) {
0935         i915_gem_object_put(obj);
0936         return vma;
0937     }
0938 
0939     err = i915_vma_pin(vma, 0, 0, PIN_USER);
0940     if (err) {
0941         i915_gem_object_put(obj);
0942         return ERR_PTR(err);
0943     }
0944 
0945     return vma;
0946 }
0947 
0948 static u32 safe_poison(u32 offset, u32 poison)
0949 {
0950     /*
0951      * Do not enable predication as it will nop all subsequent commands,
0952      * not only disabling the tests (by preventing all the other SRM) but
0953      * also preventing the arbitration events at the end of the request.
0954      */
0955     if (offset == i915_mmio_reg_offset(RING_PREDICATE_RESULT(0)))
0956         poison &= ~REG_BIT(0);
0957 
0958     return poison;
0959 }
0960 
0961 static struct i915_vma *
0962 store_context(struct intel_context *ce, struct i915_vma *scratch)
0963 {
0964     struct i915_vma *batch;
0965     u32 dw, x, *cs, *hw;
0966     u32 *defaults;
0967 
0968     batch = create_user_vma(ce->vm, SZ_64K);
0969     if (IS_ERR(batch))
0970         return batch;
0971 
0972     cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
0973     if (IS_ERR(cs)) {
0974         i915_vma_put(batch);
0975         return ERR_CAST(cs);
0976     }
0977 
0978     defaults = shmem_pin_map(ce->engine->default_state);
0979     if (!defaults) {
0980         i915_gem_object_unpin_map(batch->obj);
0981         i915_vma_put(batch);
0982         return ERR_PTR(-ENOMEM);
0983     }
0984 
0985     x = 0;
0986     dw = 0;
0987     hw = defaults;
0988     hw += LRC_STATE_OFFSET / sizeof(*hw);
0989     do {
0990         u32 len = hw[dw] & 0x7f;
0991 
0992         if (hw[dw] == 0) {
0993             dw++;
0994             continue;
0995         }
0996 
0997         if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
0998             dw += len + 2;
0999             continue;
1000         }
1001 
1002         dw++;
1003         len = (len + 1) / 2;
1004         while (len--) {
1005             *cs++ = MI_STORE_REGISTER_MEM_GEN8;
1006             *cs++ = hw[dw];
1007             *cs++ = lower_32_bits(scratch->node.start + x);
1008             *cs++ = upper_32_bits(scratch->node.start + x);
1009 
1010             dw += 2;
1011             x += 4;
1012         }
1013     } while (dw < PAGE_SIZE / sizeof(u32) &&
1014          (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1015 
1016     *cs++ = MI_BATCH_BUFFER_END;
1017 
1018     shmem_unpin_map(ce->engine->default_state, defaults);
1019 
1020     i915_gem_object_flush_map(batch->obj);
1021     i915_gem_object_unpin_map(batch->obj);
1022 
1023     return batch;
1024 }
1025 
1026 static int move_to_active(struct i915_request *rq,
1027               struct i915_vma *vma,
1028               unsigned int flags)
1029 {
1030     int err;
1031 
1032     i915_vma_lock(vma);
1033     err = i915_request_await_object(rq, vma->obj, flags);
1034     if (!err)
1035         err = i915_vma_move_to_active(vma, rq, flags);
1036     i915_vma_unlock(vma);
1037 
1038     return err;
1039 }
1040 
1041 static struct i915_request *
1042 record_registers(struct intel_context *ce,
1043          struct i915_vma *before,
1044          struct i915_vma *after,
1045          u32 *sema)
1046 {
1047     struct i915_vma *b_before, *b_after;
1048     struct i915_request *rq;
1049     u32 *cs;
1050     int err;
1051 
1052     b_before = store_context(ce, before);
1053     if (IS_ERR(b_before))
1054         return ERR_CAST(b_before);
1055 
1056     b_after = store_context(ce, after);
1057     if (IS_ERR(b_after)) {
1058         rq = ERR_CAST(b_after);
1059         goto err_before;
1060     }
1061 
1062     rq = intel_context_create_request(ce);
1063     if (IS_ERR(rq))
1064         goto err_after;
1065 
1066     err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1067     if (err)
1068         goto err_rq;
1069 
1070     err = move_to_active(rq, b_before, 0);
1071     if (err)
1072         goto err_rq;
1073 
1074     err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1075     if (err)
1076         goto err_rq;
1077 
1078     err = move_to_active(rq, b_after, 0);
1079     if (err)
1080         goto err_rq;
1081 
1082     cs = intel_ring_begin(rq, 14);
1083     if (IS_ERR(cs)) {
1084         err = PTR_ERR(cs);
1085         goto err_rq;
1086     }
1087 
1088     *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1089     *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1090     *cs++ = lower_32_bits(b_before->node.start);
1091     *cs++ = upper_32_bits(b_before->node.start);
1092 
1093     *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1094     *cs++ = MI_SEMAPHORE_WAIT |
1095         MI_SEMAPHORE_GLOBAL_GTT |
1096         MI_SEMAPHORE_POLL |
1097         MI_SEMAPHORE_SAD_NEQ_SDD;
1098     *cs++ = 0;
1099     *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1100         offset_in_page(sema);
1101     *cs++ = 0;
1102     *cs++ = MI_NOOP;
1103 
1104     *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1105     *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1106     *cs++ = lower_32_bits(b_after->node.start);
1107     *cs++ = upper_32_bits(b_after->node.start);
1108 
1109     intel_ring_advance(rq, cs);
1110 
1111     WRITE_ONCE(*sema, 0);
1112     i915_request_get(rq);
1113     i915_request_add(rq);
1114 err_after:
1115     i915_vma_put(b_after);
1116 err_before:
1117     i915_vma_put(b_before);
1118     return rq;
1119 
1120 err_rq:
1121     i915_request_add(rq);
1122     rq = ERR_PTR(err);
1123     goto err_after;
1124 }
1125 
1126 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1127 {
1128     struct i915_vma *batch;
1129     u32 dw, *cs, *hw;
1130     u32 *defaults;
1131 
1132     batch = create_user_vma(ce->vm, SZ_64K);
1133     if (IS_ERR(batch))
1134         return batch;
1135 
1136     cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1137     if (IS_ERR(cs)) {
1138         i915_vma_put(batch);
1139         return ERR_CAST(cs);
1140     }
1141 
1142     defaults = shmem_pin_map(ce->engine->default_state);
1143     if (!defaults) {
1144         i915_gem_object_unpin_map(batch->obj);
1145         i915_vma_put(batch);
1146         return ERR_PTR(-ENOMEM);
1147     }
1148 
1149     dw = 0;
1150     hw = defaults;
1151     hw += LRC_STATE_OFFSET / sizeof(*hw);
1152     do {
1153         u32 len = hw[dw] & 0x7f;
1154 
1155         if (hw[dw] == 0) {
1156             dw++;
1157             continue;
1158         }
1159 
1160         if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1161             dw += len + 2;
1162             continue;
1163         }
1164 
1165         dw++;
1166         len = (len + 1) / 2;
1167         *cs++ = MI_LOAD_REGISTER_IMM(len);
1168         while (len--) {
1169             *cs++ = hw[dw];
1170             *cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine,
1171                                   MI_LRI_LRM_CS_MMIO),
1172                         poison);
1173             dw += 2;
1174         }
1175     } while (dw < PAGE_SIZE / sizeof(u32) &&
1176          (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1177 
1178     *cs++ = MI_BATCH_BUFFER_END;
1179 
1180     shmem_unpin_map(ce->engine->default_state, defaults);
1181 
1182     i915_gem_object_flush_map(batch->obj);
1183     i915_gem_object_unpin_map(batch->obj);
1184 
1185     return batch;
1186 }
1187 
1188 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1189 {
1190     struct i915_request *rq;
1191     struct i915_vma *batch;
1192     u32 *cs;
1193     int err;
1194 
1195     batch = load_context(ce, poison);
1196     if (IS_ERR(batch))
1197         return PTR_ERR(batch);
1198 
1199     rq = intel_context_create_request(ce);
1200     if (IS_ERR(rq)) {
1201         err = PTR_ERR(rq);
1202         goto err_batch;
1203     }
1204 
1205     err = move_to_active(rq, batch, 0);
1206     if (err)
1207         goto err_rq;
1208 
1209     cs = intel_ring_begin(rq, 8);
1210     if (IS_ERR(cs)) {
1211         err = PTR_ERR(cs);
1212         goto err_rq;
1213     }
1214 
1215     *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1216     *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1217     *cs++ = lower_32_bits(batch->node.start);
1218     *cs++ = upper_32_bits(batch->node.start);
1219 
1220     *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1221     *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1222         offset_in_page(sema);
1223     *cs++ = 0;
1224     *cs++ = 1;
1225 
1226     intel_ring_advance(rq, cs);
1227 
1228     rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1229 err_rq:
1230     i915_request_add(rq);
1231 err_batch:
1232     i915_vma_put(batch);
1233     return err;
1234 }
1235 
1236 static bool is_moving(u32 a, u32 b)
1237 {
1238     return a != b;
1239 }
1240 
1241 static int compare_isolation(struct intel_engine_cs *engine,
1242                  struct i915_vma *ref[2],
1243                  struct i915_vma *result[2],
1244                  struct intel_context *ce,
1245                  u32 poison)
1246 {
1247     u32 x, dw, *hw, *lrc;
1248     u32 *A[2], *B[2];
1249     u32 *defaults;
1250     int err = 0;
1251 
1252     A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1253     if (IS_ERR(A[0]))
1254         return PTR_ERR(A[0]);
1255 
1256     A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1257     if (IS_ERR(A[1])) {
1258         err = PTR_ERR(A[1]);
1259         goto err_A0;
1260     }
1261 
1262     B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1263     if (IS_ERR(B[0])) {
1264         err = PTR_ERR(B[0]);
1265         goto err_A1;
1266     }
1267 
1268     B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1269     if (IS_ERR(B[1])) {
1270         err = PTR_ERR(B[1]);
1271         goto err_B0;
1272     }
1273 
1274     lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1275                            i915_coherent_map_type(engine->i915,
1276                                       ce->state->obj,
1277                                       false));
1278     if (IS_ERR(lrc)) {
1279         err = PTR_ERR(lrc);
1280         goto err_B1;
1281     }
1282     lrc += LRC_STATE_OFFSET / sizeof(*hw);
1283 
1284     defaults = shmem_pin_map(ce->engine->default_state);
1285     if (!defaults) {
1286         err = -ENOMEM;
1287         goto err_lrc;
1288     }
1289 
1290     x = 0;
1291     dw = 0;
1292     hw = defaults;
1293     hw += LRC_STATE_OFFSET / sizeof(*hw);
1294     do {
1295         u32 len = hw[dw] & 0x7f;
1296 
1297         if (hw[dw] == 0) {
1298             dw++;
1299             continue;
1300         }
1301 
1302         if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1303             dw += len + 2;
1304             continue;
1305         }
1306 
1307         dw++;
1308         len = (len + 1) / 2;
1309         while (len--) {
1310             if (!is_moving(A[0][x], A[1][x]) &&
1311                 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1312                 switch (hw[dw] & 4095) {
1313                 case 0x30: /* RING_HEAD */
1314                 case 0x34: /* RING_TAIL */
1315                     break;
1316 
1317                 default:
1318                     pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1319                            engine->name, dw,
1320                            hw[dw], hw[dw + 1],
1321                            A[0][x], B[0][x], B[1][x],
1322                            poison, lrc[dw + 1]);
1323                     err = -EINVAL;
1324                 }
1325             }
1326             dw += 2;
1327             x++;
1328         }
1329     } while (dw < PAGE_SIZE / sizeof(u32) &&
1330          (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1331 
1332     shmem_unpin_map(ce->engine->default_state, defaults);
1333 err_lrc:
1334     i915_gem_object_unpin_map(ce->state->obj);
1335 err_B1:
1336     i915_gem_object_unpin_map(result[1]->obj);
1337 err_B0:
1338     i915_gem_object_unpin_map(result[0]->obj);
1339 err_A1:
1340     i915_gem_object_unpin_map(ref[1]->obj);
1341 err_A0:
1342     i915_gem_object_unpin_map(ref[0]->obj);
1343     return err;
1344 }
1345 
1346 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1347 {
1348     u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1349     struct i915_vma *ref[2], *result[2];
1350     struct intel_context *A, *B;
1351     struct i915_request *rq;
1352     int err;
1353 
1354     A = intel_context_create(engine);
1355     if (IS_ERR(A))
1356         return PTR_ERR(A);
1357 
1358     B = intel_context_create(engine);
1359     if (IS_ERR(B)) {
1360         err = PTR_ERR(B);
1361         goto err_A;
1362     }
1363 
1364     ref[0] = create_user_vma(A->vm, SZ_64K);
1365     if (IS_ERR(ref[0])) {
1366         err = PTR_ERR(ref[0]);
1367         goto err_B;
1368     }
1369 
1370     ref[1] = create_user_vma(A->vm, SZ_64K);
1371     if (IS_ERR(ref[1])) {
1372         err = PTR_ERR(ref[1]);
1373         goto err_ref0;
1374     }
1375 
1376     rq = record_registers(A, ref[0], ref[1], sema);
1377     if (IS_ERR(rq)) {
1378         err = PTR_ERR(rq);
1379         goto err_ref1;
1380     }
1381 
1382     WRITE_ONCE(*sema, 1);
1383     wmb();
1384 
1385     if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1386         i915_request_put(rq);
1387         err = -ETIME;
1388         goto err_ref1;
1389     }
1390     i915_request_put(rq);
1391 
1392     result[0] = create_user_vma(A->vm, SZ_64K);
1393     if (IS_ERR(result[0])) {
1394         err = PTR_ERR(result[0]);
1395         goto err_ref1;
1396     }
1397 
1398     result[1] = create_user_vma(A->vm, SZ_64K);
1399     if (IS_ERR(result[1])) {
1400         err = PTR_ERR(result[1]);
1401         goto err_result0;
1402     }
1403 
1404     rq = record_registers(A, result[0], result[1], sema);
1405     if (IS_ERR(rq)) {
1406         err = PTR_ERR(rq);
1407         goto err_result1;
1408     }
1409 
1410     err = poison_registers(B, poison, sema);
1411     if (err) {
1412         WRITE_ONCE(*sema, -1);
1413         i915_request_put(rq);
1414         goto err_result1;
1415     }
1416 
1417     if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1418         i915_request_put(rq);
1419         err = -ETIME;
1420         goto err_result1;
1421     }
1422     i915_request_put(rq);
1423 
1424     err = compare_isolation(engine, ref, result, A, poison);
1425 
1426 err_result1:
1427     i915_vma_put(result[1]);
1428 err_result0:
1429     i915_vma_put(result[0]);
1430 err_ref1:
1431     i915_vma_put(ref[1]);
1432 err_ref0:
1433     i915_vma_put(ref[0]);
1434 err_B:
1435     intel_context_put(B);
1436 err_A:
1437     intel_context_put(A);
1438     return err;
1439 }
1440 
1441 static bool skip_isolation(const struct intel_engine_cs *engine)
1442 {
1443     if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)
1444         return true;
1445 
1446     if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)
1447         return true;
1448 
1449     return false;
1450 }
1451 
1452 static int live_lrc_isolation(void *arg)
1453 {
1454     struct intel_gt *gt = arg;
1455     struct intel_engine_cs *engine;
1456     enum intel_engine_id id;
1457     const u32 poison[] = {
1458         STACK_MAGIC,
1459         0x3a3a3a3a,
1460         0x5c5c5c5c,
1461         0xffffffff,
1462         0xffff0000,
1463     };
1464     int err = 0;
1465 
1466     /*
1467      * Our goal is try and verify that per-context state cannot be
1468      * tampered with by another non-privileged client.
1469      *
1470      * We take the list of context registers from the LRI in the default
1471      * context image and attempt to modify that list from a remote context.
1472      */
1473 
1474     for_each_engine(engine, gt, id) {
1475         int i;
1476 
1477         /* Just don't even ask */
1478         if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1479             skip_isolation(engine))
1480             continue;
1481 
1482         intel_engine_pm_get(engine);
1483         for (i = 0; i < ARRAY_SIZE(poison); i++) {
1484             int result;
1485 
1486             result = __lrc_isolation(engine, poison[i]);
1487             if (result && !err)
1488                 err = result;
1489 
1490             result = __lrc_isolation(engine, ~poison[i]);
1491             if (result && !err)
1492                 err = result;
1493         }
1494         intel_engine_pm_put(engine);
1495         if (igt_flush_test(gt->i915)) {
1496             err = -EIO;
1497             break;
1498         }
1499     }
1500 
1501     return err;
1502 }
1503 
1504 static int indirect_ctx_submit_req(struct intel_context *ce)
1505 {
1506     struct i915_request *rq;
1507     int err = 0;
1508 
1509     rq = intel_context_create_request(ce);
1510     if (IS_ERR(rq))
1511         return PTR_ERR(rq);
1512 
1513     i915_request_get(rq);
1514     i915_request_add(rq);
1515 
1516     if (i915_request_wait(rq, 0, HZ / 5) < 0)
1517         err = -ETIME;
1518 
1519     i915_request_put(rq);
1520 
1521     return err;
1522 }
1523 
1524 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1525 #define CTX_BB_CANARY_INDEX  (CTX_BB_CANARY_OFFSET / sizeof(u32))
1526 
1527 static u32 *
1528 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1529 {
1530     *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1531         MI_SRM_LRM_GLOBAL_GTT |
1532         MI_LRI_LRM_CS_MMIO;
1533     *cs++ = i915_mmio_reg_offset(RING_START(0));
1534     *cs++ = i915_ggtt_offset(ce->state) +
1535         context_wa_bb_offset(ce) +
1536         CTX_BB_CANARY_OFFSET;
1537     *cs++ = 0;
1538 
1539     return cs;
1540 }
1541 
1542 static void
1543 indirect_ctx_bb_setup(struct intel_context *ce)
1544 {
1545     u32 *cs = context_indirect_bb(ce);
1546 
1547     cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1548 
1549     setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1550 }
1551 
1552 static bool check_ring_start(struct intel_context *ce)
1553 {
1554     const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1555         LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1556 
1557     if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1558         return true;
1559 
1560     pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1561            ctx_bb[CTX_BB_CANARY_INDEX],
1562            ce->lrc_reg_state[CTX_RING_START]);
1563 
1564     return false;
1565 }
1566 
1567 static int indirect_ctx_bb_check(struct intel_context *ce)
1568 {
1569     int err;
1570 
1571     err = indirect_ctx_submit_req(ce);
1572     if (err)
1573         return err;
1574 
1575     if (!check_ring_start(ce))
1576         return -EINVAL;
1577 
1578     return 0;
1579 }
1580 
1581 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1582 {
1583     struct intel_context *a, *b;
1584     int err;
1585 
1586     a = intel_context_create(engine);
1587     if (IS_ERR(a))
1588         return PTR_ERR(a);
1589     err = intel_context_pin(a);
1590     if (err)
1591         goto put_a;
1592 
1593     b = intel_context_create(engine);
1594     if (IS_ERR(b)) {
1595         err = PTR_ERR(b);
1596         goto unpin_a;
1597     }
1598     err = intel_context_pin(b);
1599     if (err)
1600         goto put_b;
1601 
1602     /* We use the already reserved extra page in context state */
1603     if (!a->wa_bb_page) {
1604         GEM_BUG_ON(b->wa_bb_page);
1605         GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);
1606         goto unpin_b;
1607     }
1608 
1609     /*
1610      * In order to test that our per context bb is truly per context,
1611      * and executes at the intended spot on context restoring process,
1612      * make the batch store the ring start value to memory.
1613      * As ring start is restored apriori of starting the indirect ctx bb and
1614      * as it will be different for each context, it fits to this purpose.
1615      */
1616     indirect_ctx_bb_setup(a);
1617     indirect_ctx_bb_setup(b);
1618 
1619     err = indirect_ctx_bb_check(a);
1620     if (err)
1621         goto unpin_b;
1622 
1623     err = indirect_ctx_bb_check(b);
1624 
1625 unpin_b:
1626     intel_context_unpin(b);
1627 put_b:
1628     intel_context_put(b);
1629 unpin_a:
1630     intel_context_unpin(a);
1631 put_a:
1632     intel_context_put(a);
1633 
1634     return err;
1635 }
1636 
1637 static int live_lrc_indirect_ctx_bb(void *arg)
1638 {
1639     struct intel_gt *gt = arg;
1640     struct intel_engine_cs *engine;
1641     enum intel_engine_id id;
1642     int err = 0;
1643 
1644     for_each_engine(engine, gt, id) {
1645         intel_engine_pm_get(engine);
1646         err = __live_lrc_indirect_ctx_bb(engine);
1647         intel_engine_pm_put(engine);
1648 
1649         if (igt_flush_test(gt->i915))
1650             err = -EIO;
1651 
1652         if (err)
1653             break;
1654     }
1655 
1656     return err;
1657 }
1658 
1659 static void garbage_reset(struct intel_engine_cs *engine,
1660               struct i915_request *rq)
1661 {
1662     const unsigned int bit = I915_RESET_ENGINE + engine->id;
1663     unsigned long *lock = &engine->gt->reset.flags;
1664 
1665     local_bh_disable();
1666     if (!test_and_set_bit(bit, lock)) {
1667         tasklet_disable(&engine->sched_engine->tasklet);
1668 
1669         if (!rq->fence.error)
1670             __intel_engine_reset_bh(engine, NULL);
1671 
1672         tasklet_enable(&engine->sched_engine->tasklet);
1673         clear_and_wake_up_bit(bit, lock);
1674     }
1675     local_bh_enable();
1676 }
1677 
1678 static struct i915_request *garbage(struct intel_context *ce,
1679                     struct rnd_state *prng)
1680 {
1681     struct i915_request *rq;
1682     int err;
1683 
1684     err = intel_context_pin(ce);
1685     if (err)
1686         return ERR_PTR(err);
1687 
1688     prandom_bytes_state(prng,
1689                 ce->lrc_reg_state,
1690                 ce->engine->context_size -
1691                 LRC_STATE_OFFSET);
1692 
1693     rq = intel_context_create_request(ce);
1694     if (IS_ERR(rq)) {
1695         err = PTR_ERR(rq);
1696         goto err_unpin;
1697     }
1698 
1699     i915_request_get(rq);
1700     i915_request_add(rq);
1701     return rq;
1702 
1703 err_unpin:
1704     intel_context_unpin(ce);
1705     return ERR_PTR(err);
1706 }
1707 
1708 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1709 {
1710     struct intel_context *ce;
1711     struct i915_request *hang;
1712     int err = 0;
1713 
1714     ce = intel_context_create(engine);
1715     if (IS_ERR(ce))
1716         return PTR_ERR(ce);
1717 
1718     hang = garbage(ce, prng);
1719     if (IS_ERR(hang)) {
1720         err = PTR_ERR(hang);
1721         goto err_ce;
1722     }
1723 
1724     if (wait_for_submit(engine, hang, HZ / 2)) {
1725         i915_request_put(hang);
1726         err = -ETIME;
1727         goto err_ce;
1728     }
1729 
1730     intel_context_set_banned(ce);
1731     garbage_reset(engine, hang);
1732 
1733     intel_engine_flush_submission(engine);
1734     if (!hang->fence.error) {
1735         i915_request_put(hang);
1736         pr_err("%s: corrupted context was not reset\n",
1737                engine->name);
1738         err = -EINVAL;
1739         goto err_ce;
1740     }
1741 
1742     if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1743         pr_err("%s: corrupted context did not recover\n",
1744                engine->name);
1745         i915_request_put(hang);
1746         err = -EIO;
1747         goto err_ce;
1748     }
1749     i915_request_put(hang);
1750 
1751 err_ce:
1752     intel_context_put(ce);
1753     return err;
1754 }
1755 
1756 static int live_lrc_garbage(void *arg)
1757 {
1758     struct intel_gt *gt = arg;
1759     struct intel_engine_cs *engine;
1760     enum intel_engine_id id;
1761 
1762     /*
1763      * Verify that we can recover if one context state is completely
1764      * corrupted.
1765      */
1766 
1767     if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1768         return 0;
1769 
1770     for_each_engine(engine, gt, id) {
1771         I915_RND_STATE(prng);
1772         int err = 0, i;
1773 
1774         if (!intel_has_reset_engine(engine->gt))
1775             continue;
1776 
1777         intel_engine_pm_get(engine);
1778         for (i = 0; i < 3; i++) {
1779             err = __lrc_garbage(engine, &prng);
1780             if (err)
1781                 break;
1782         }
1783         intel_engine_pm_put(engine);
1784 
1785         if (igt_flush_test(gt->i915))
1786             err = -EIO;
1787         if (err)
1788             return err;
1789     }
1790 
1791     return 0;
1792 }
1793 
1794 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1795 {
1796     struct intel_context *ce;
1797     struct i915_request *rq;
1798     IGT_TIMEOUT(end_time);
1799     int err;
1800 
1801     ce = intel_context_create(engine);
1802     if (IS_ERR(ce))
1803         return PTR_ERR(ce);
1804 
1805     ce->stats.runtime.num_underflow = 0;
1806     ce->stats.runtime.max_underflow = 0;
1807 
1808     do {
1809         unsigned int loop = 1024;
1810 
1811         while (loop) {
1812             rq = intel_context_create_request(ce);
1813             if (IS_ERR(rq)) {
1814                 err = PTR_ERR(rq);
1815                 goto err_rq;
1816             }
1817 
1818             if (--loop == 0)
1819                 i915_request_get(rq);
1820 
1821             i915_request_add(rq);
1822         }
1823 
1824         if (__igt_timeout(end_time, NULL))
1825             break;
1826 
1827         i915_request_put(rq);
1828     } while (1);
1829 
1830     err = i915_request_wait(rq, 0, HZ / 5);
1831     if (err < 0) {
1832         pr_err("%s: request not completed!\n", engine->name);
1833         goto err_wait;
1834     }
1835 
1836     igt_flush_test(engine->i915);
1837 
1838     pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1839         engine->name,
1840         intel_context_get_total_runtime_ns(ce),
1841         intel_context_get_avg_runtime_ns(ce));
1842 
1843     err = 0;
1844     if (ce->stats.runtime.num_underflow) {
1845         pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1846                engine->name,
1847                ce->stats.runtime.num_underflow,
1848                ce->stats.runtime.max_underflow);
1849         GEM_TRACE_DUMP();
1850         err = -EOVERFLOW;
1851     }
1852 
1853 err_wait:
1854     i915_request_put(rq);
1855 err_rq:
1856     intel_context_put(ce);
1857     return err;
1858 }
1859 
1860 static int live_pphwsp_runtime(void *arg)
1861 {
1862     struct intel_gt *gt = arg;
1863     struct intel_engine_cs *engine;
1864     enum intel_engine_id id;
1865     int err = 0;
1866 
1867     /*
1868      * Check that cumulative context runtime as stored in the pphwsp[16]
1869      * is monotonic.
1870      */
1871 
1872     for_each_engine(engine, gt, id) {
1873         err = __live_pphwsp_runtime(engine);
1874         if (err)
1875             break;
1876     }
1877 
1878     if (igt_flush_test(gt->i915))
1879         err = -EIO;
1880 
1881     return err;
1882 }
1883 
1884 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1885 {
1886     static const struct i915_subtest tests[] = {
1887         SUBTEST(live_lrc_layout),
1888         SUBTEST(live_lrc_fixed),
1889         SUBTEST(live_lrc_state),
1890         SUBTEST(live_lrc_gpr),
1891         SUBTEST(live_lrc_isolation),
1892         SUBTEST(live_lrc_timestamp),
1893         SUBTEST(live_lrc_garbage),
1894         SUBTEST(live_pphwsp_runtime),
1895         SUBTEST(live_lrc_indirect_ctx_bb),
1896     };
1897 
1898     if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1899         return 0;
1900 
1901     return intel_gt_live_subtests(tests, to_gt(i915));
1902 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!