i915/gt/gen7_renderclear.c

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2019 Intel Corporation
0004  */
0005
0006 #include "gen7_renderclear.h"
0007 #include "i915_drv.h"
0008 #include "intel_gpu_commands.h"
0009 #include "intel_gt_regs.h"
0010
0011 #define GT3_INLINE_DATA_DELAYS 0x1E00
0012 #define batch_advance(Y, CS) GEM_BUG_ON((Y)->end != (CS))
0013
0014 struct cb_kernel {
0015     const void *data;
0016     u32 size;
0017 };
0018
0019 #define CB_KERNEL(name) { .data = (name), .size = sizeof(name) }
0020
0021 #include "ivb_clear_kernel.c"
0022 static const struct cb_kernel cb_kernel_ivb = CB_KERNEL(ivb_clear_kernel);
0023
0024 #include "hsw_clear_kernel.c"
0025 static const struct cb_kernel cb_kernel_hsw = CB_KERNEL(hsw_clear_kernel);
0026
0027 struct batch_chunk {
0028     struct i915_vma *vma;
0029     u32 offset;
0030     u32 *start;
0031     u32 *end;
0032     u32 max_items;
0033 };
0034
0035 struct batch_vals {
0036     u32 max_threads;
0037     u32 state_start;
0038     u32 surface_start;
0039     u32 surface_height;
0040     u32 surface_width;
0041     u32 size;
0042 };
0043
0044 static int num_primitives(const struct batch_vals *bv)
0045 {
0046     /*
0047      * We need to saturate the GPU with work in order to dispatch
0048      * a shader on every HW thread, and clear the thread-local registers.
0049      * In short, we have to dispatch work faster than the shaders can
0050      * run in order to fill the EU and occupy each HW thread.
0051      */
0052     return bv->max_threads;
0053 }
0054
0055 static void
0056 batch_get_defaults(struct drm_i915_private *i915, struct batch_vals *bv)
0057 {
0058     if (IS_HASWELL(i915)) {
0059         switch (INTEL_INFO(i915)->gt) {
0060         default:
0061         case 1:
0062             bv->max_threads = 70;
0063             break;
0064         case 2:
0065             bv->max_threads = 140;
0066             break;
0067         case 3:
0068             bv->max_threads = 280;
0069             break;
0070         }
0071         bv->surface_height = 16 * 16;
0072         bv->surface_width = 32 * 2 * 16;
0073     } else {
0074         switch (INTEL_INFO(i915)->gt) {
0075         default:
0076         case 1: /* including vlv */
0077             bv->max_threads = 36;
0078             break;
0079         case 2:
0080             bv->max_threads = 128;
0081             break;
0082         }
0083         bv->surface_height = 16 * 8;
0084         bv->surface_width = 32 * 16;
0085     }
0086     bv->state_start = round_up(SZ_1K + num_primitives(bv) * 64, SZ_4K);
0087     bv->surface_start = bv->state_start + SZ_4K;
0088     bv->size = bv->surface_start + bv->surface_height * bv->surface_width;
0089 }
0090
0091 static void batch_init(struct batch_chunk *bc,
0092                struct i915_vma *vma,
0093                u32 *start, u32 offset, u32 max_bytes)
0094 {
0095     bc->vma = vma;
0096     bc->offset = offset;
0097     bc->start = start + bc->offset / sizeof(*bc->start);
0098     bc->end = bc->start;
0099     bc->max_items = max_bytes / sizeof(*bc->start);
0100 }
0101
0102 static u32 batch_offset(const struct batch_chunk *bc, u32 *cs)
0103 {
0104     return (cs - bc->start) * sizeof(*bc->start) + bc->offset;
0105 }
0106
0107 static u32 batch_addr(const struct batch_chunk *bc)
0108 {
0109     return bc->vma->node.start;
0110 }
0111
0112 static void batch_add(struct batch_chunk *bc, const u32 d)
0113 {
0114     GEM_BUG_ON((bc->end - bc->start) >= bc->max_items);
0115     *bc->end++ = d;
0116 }
0117
0118 static u32 *batch_alloc_items(struct batch_chunk *bc, u32 align, u32 items)
0119 {
0120     u32 *map;
0121
0122     if (align) {
0123         u32 *end = PTR_ALIGN(bc->end, align);
0124
0125         memset32(bc->end, 0, end - bc->end);
0126         bc->end = end;
0127     }
0128
0129     map = bc->end;
0130     bc->end += items;
0131
0132     return map;
0133 }
0134
0135 static u32 *batch_alloc_bytes(struct batch_chunk *bc, u32 align, u32 bytes)
0136 {
0137     GEM_BUG_ON(!IS_ALIGNED(bytes, sizeof(*bc->start)));
0138     return batch_alloc_items(bc, align, bytes / sizeof(*bc->start));
0139 }
0140
0141 static u32
0142 gen7_fill_surface_state(struct batch_chunk *state,
0143             const u32 dst_offset,
0144             const struct batch_vals *bv)
0145 {
0146     u32 surface_h = bv->surface_height;
0147     u32 surface_w = bv->surface_width;
0148     u32 *cs = batch_alloc_items(state, 32, 8);
0149     u32 offset = batch_offset(state, cs);
0150
0151 #define SURFACE_2D 1
0152 #define SURFACEFORMAT_B8G8R8A8_UNORM 0x0C0
0153 #define RENDER_CACHE_READ_WRITE 1
0154
0155     *cs++ = SURFACE_2D << 29 |
0156         (SURFACEFORMAT_B8G8R8A8_UNORM << 18) |
0157         (RENDER_CACHE_READ_WRITE << 8);
0158
0159     *cs++ = batch_addr(state) + dst_offset;
0160
0161     *cs++ = ((surface_h / 4 - 1) << 16) | (surface_w / 4 - 1);
0162     *cs++ = surface_w;
0163     *cs++ = 0;
0164     *cs++ = 0;
0165     *cs++ = 0;
0166 #define SHADER_CHANNELS(r, g, b, a) \
0167     (((r) << 25) | ((g) << 22) | ((b) << 19) | ((a) << 16))
0168     *cs++ = SHADER_CHANNELS(4, 5, 6, 7);
0169     batch_advance(state, cs);
0170
0171     return offset;
0172 }
0173
0174 static u32
0175 gen7_fill_binding_table(struct batch_chunk *state,
0176             const struct batch_vals *bv)
0177 {
0178     u32 surface_start =
0179         gen7_fill_surface_state(state, bv->surface_start, bv);
0180     u32 *cs = batch_alloc_items(state, 32, 8);
0181     u32 offset = batch_offset(state, cs);
0182
0183     *cs++ = surface_start - state->offset;
0184     *cs++ = 0;
0185     *cs++ = 0;
0186     *cs++ = 0;
0187     *cs++ = 0;
0188     *cs++ = 0;
0189     *cs++ = 0;
0190     *cs++ = 0;
0191     batch_advance(state, cs);
0192
0193     return offset;
0194 }
0195
0196 static u32
0197 gen7_fill_kernel_data(struct batch_chunk *state,
0198               const u32 *data,
0199               const u32 size)
0200 {
0201     return batch_offset(state,
0202                 memcpy(batch_alloc_bytes(state, 64, size),
0203                    data, size));
0204 }
0205
0206 static u32
0207 gen7_fill_interface_descriptor(struct batch_chunk *state,
0208                    const struct batch_vals *bv,
0209                    const struct cb_kernel *kernel,
0210                    unsigned int count)
0211 {
0212     u32 kernel_offset =
0213         gen7_fill_kernel_data(state, kernel->data, kernel->size);
0214     u32 binding_table = gen7_fill_binding_table(state, bv);
0215     u32 *cs = batch_alloc_items(state, 32, 8 * count);
0216     u32 offset = batch_offset(state, cs);
0217
0218     *cs++ = kernel_offset;
0219     *cs++ = (1 << 7) | (1 << 13);
0220     *cs++ = 0;
0221     *cs++ = (binding_table - state->offset) | 1;
0222     *cs++ = 0;
0223     *cs++ = 0;
0224     *cs++ = 0;
0225     *cs++ = 0;
0226
0227     /* 1 - 63dummy idds */
0228     memset32(cs, 0x00, (count - 1) * 8);
0229     batch_advance(state, cs + (count - 1) * 8);
0230
0231     return offset;
0232 }
0233
0234 static void
0235 gen7_emit_state_base_address(struct batch_chunk *batch,
0236                  u32 surface_state_base)
0237 {
0238     u32 *cs = batch_alloc_items(batch, 0, 10);
0239
0240     *cs++ = STATE_BASE_ADDRESS | (10 - 2);
0241     /* general */
0242     *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
0243     /* surface */
0244     *cs++ = (batch_addr(batch) + surface_state_base) | BASE_ADDRESS_MODIFY;
0245     /* dynamic */
0246     *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
0247     /* indirect */
0248     *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
0249     /* instruction */
0250     *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
0251
0252     /* general/dynamic/indirect/instruction access Bound */
0253     *cs++ = 0;
0254     *cs++ = BASE_ADDRESS_MODIFY;
0255     *cs++ = 0;
0256     *cs++ = BASE_ADDRESS_MODIFY;
0257     batch_advance(batch, cs);
0258 }
0259
0260 static void
0261 gen7_emit_vfe_state(struct batch_chunk *batch,
0262             const struct batch_vals *bv,
0263             u32 urb_size, u32 curbe_size,
0264             u32 mode)
0265 {
0266     u32 threads = bv->max_threads - 1;
0267     u32 *cs = batch_alloc_items(batch, 32, 8);
0268
0269     *cs++ = MEDIA_VFE_STATE | (8 - 2);
0270
0271     /* scratch buffer */
0272     *cs++ = 0;
0273
0274     /* number of threads & urb entries for GPGPU vs Media Mode */
0275     *cs++ = threads << 16 | 1 << 8 | mode << 2;
0276
0277     *cs++ = 0;
0278
0279     /* urb entry size & curbe size in 256 bits unit */
0280     *cs++ = urb_size << 16 | curbe_size;
0281
0282     /* scoreboard */
0283     *cs++ = 0;
0284     *cs++ = 0;
0285     *cs++ = 0;
0286     batch_advance(batch, cs);
0287 }
0288
0289 static void
0290 gen7_emit_interface_descriptor_load(struct batch_chunk *batch,
0291                     const u32 interface_descriptor,
0292                     unsigned int count)
0293 {
0294     u32 *cs = batch_alloc_items(batch, 8, 4);
0295
0296     *cs++ = MEDIA_INTERFACE_DESCRIPTOR_LOAD | (4 - 2);
0297     *cs++ = 0;
0298     *cs++ = count * 8 * sizeof(*cs);
0299
0300     /*
0301      * interface descriptor address - it is relative to the dynamics base
0302      * address
0303      */
0304     *cs++ = interface_descriptor;
0305     batch_advance(batch, cs);
0306 }
0307
0308 static void
0309 gen7_emit_media_object(struct batch_chunk *batch,
0310                unsigned int media_object_index)
0311 {
0312     unsigned int x_offset = (media_object_index % 16) * 64;
0313     unsigned int y_offset = (media_object_index / 16) * 16;
0314     unsigned int pkt = 6 + 3;
0315     u32 *cs;
0316
0317     cs = batch_alloc_items(batch, 8, pkt);
0318
0319     *cs++ = MEDIA_OBJECT | (pkt - 2);
0320
0321     /* interface descriptor offset */
0322     *cs++ = 0;
0323
0324     /* without indirect data */
0325     *cs++ = 0;
0326     *cs++ = 0;
0327
0328     /* scoreboard */
0329     *cs++ = 0;
0330     *cs++ = 0;
0331
0332     /* inline */
0333     *cs++ = y_offset << 16 | x_offset;
0334     *cs++ = 0;
0335     *cs++ = GT3_INLINE_DATA_DELAYS;
0336
0337     batch_advance(batch, cs);
0338 }
0339
0340 static void gen7_emit_pipeline_flush(struct batch_chunk *batch)
0341 {
0342     u32 *cs = batch_alloc_items(batch, 0, 4);
0343
0344     *cs++ = GFX_OP_PIPE_CONTROL(4);
0345     *cs++ = PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
0346         PIPE_CONTROL_DEPTH_CACHE_FLUSH |
0347         PIPE_CONTROL_DC_FLUSH_ENABLE |
0348         PIPE_CONTROL_CS_STALL;
0349     *cs++ = 0;
0350     *cs++ = 0;
0351
0352     batch_advance(batch, cs);
0353 }
0354
0355 static void gen7_emit_pipeline_invalidate(struct batch_chunk *batch)
0356 {
0357     u32 *cs = batch_alloc_items(batch, 0, 10);
0358
0359     /* ivb: Stall before STATE_CACHE_INVALIDATE */
0360     *cs++ = GFX_OP_PIPE_CONTROL(5);
0361     *cs++ = PIPE_CONTROL_STALL_AT_SCOREBOARD |
0362         PIPE_CONTROL_CS_STALL;
0363     *cs++ = 0;
0364     *cs++ = 0;
0365     *cs++ = 0;
0366
0367     *cs++ = GFX_OP_PIPE_CONTROL(5);
0368     *cs++ = PIPE_CONTROL_STATE_CACHE_INVALIDATE;
0369     *cs++ = 0;
0370     *cs++ = 0;
0371     *cs++ = 0;
0372
0373     batch_advance(batch, cs);
0374 }
0375
0376 static void emit_batch(struct i915_vma * const vma,
0377                u32 *start,
0378                const struct batch_vals *bv)
0379 {
0380     struct drm_i915_private *i915 = vma->vm->i915;
0381     const unsigned int desc_count = 1;
0382     const unsigned int urb_size = 1;
0383     struct batch_chunk cmds, state;
0384     u32 descriptors;
0385     unsigned int i;
0386
0387     batch_init(&cmds, vma, start, 0, bv->state_start);
0388     batch_init(&state, vma, start, bv->state_start, SZ_4K);
0389
0390     descriptors = gen7_fill_interface_descriptor(&state, bv,
0391                              IS_HASWELL(i915) ?
0392                              &cb_kernel_hsw :
0393                              &cb_kernel_ivb,
0394                              desc_count);
0395
0396     /* Reset inherited context registers */
0397     gen7_emit_pipeline_flush(&cmds);
0398     gen7_emit_pipeline_invalidate(&cmds);
0399     batch_add(&cmds, MI_LOAD_REGISTER_IMM(2));
0400     batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_0_GEN7));
0401     batch_add(&cmds, 0xffff0000 |
0402             ((IS_IVB_GT1(i915) || IS_VALLEYVIEW(i915)) ?
0403              HIZ_RAW_STALL_OPT_DISABLE :
0404              0));
0405     batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_1));
0406     batch_add(&cmds, 0xffff0000 | PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
0407     gen7_emit_pipeline_invalidate(&cmds);
0408     gen7_emit_pipeline_flush(&cmds);
0409
0410     /* Switch to the media pipeline and our base address */
0411     gen7_emit_pipeline_invalidate(&cmds);
0412     batch_add(&cmds, PIPELINE_SELECT | PIPELINE_SELECT_MEDIA);
0413     batch_add(&cmds, MI_NOOP);
0414     gen7_emit_pipeline_invalidate(&cmds);
0415
0416     gen7_emit_pipeline_flush(&cmds);
0417     gen7_emit_state_base_address(&cmds, descriptors);
0418     gen7_emit_pipeline_invalidate(&cmds);
0419
0420     /* Set the clear-residual kernel state */
0421     gen7_emit_vfe_state(&cmds, bv, urb_size - 1, 0, 0);
0422     gen7_emit_interface_descriptor_load(&cmds, descriptors, desc_count);
0423
0424     /* Execute the kernel on all HW threads */
0425     for (i = 0; i < num_primitives(bv); i++)
0426         gen7_emit_media_object(&cmds, i);
0427
0428     batch_add(&cmds, MI_BATCH_BUFFER_END);
0429 }
0430
0431 int gen7_setup_clear_gpr_bb(struct intel_engine_cs * const engine,
0432                 struct i915_vma * const vma)
0433 {
0434     struct batch_vals bv;
0435     u32 *batch;
0436
0437     batch_get_defaults(engine->i915, &bv);
0438     if (!vma)
0439         return bv.size;
0440
0441     GEM_BUG_ON(vma->obj->base.size < bv.size);
0442
0443     batch = i915_gem_object_pin_map(vma->obj, I915_MAP_WC);
0444     if (IS_ERR(batch))
0445         return PTR_ERR(batch);
0446
0447     emit_batch(vma, memset(batch, 0, bv.size), &bv);
0448
0449     i915_gem_object_flush_map(vma->obj);
0450     __i915_gem_object_release_map(vma->obj);
0451
0452     return 0;
0453 }