Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice (including the next
0012  * paragraph) shall be included in all copies or substantial portions of the
0013  * Software.
0014  *
0015  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0016  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0017  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0018  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
0019  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
0020  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
0021  * SOFTWARE.
0022  *
0023  * Authors:
0024  *    Zhiyuan Lv <zhiyuan.lv@intel.com>
0025  *    Zhi Wang <zhi.a.wang@intel.com>
0026  *
0027  * Contributors:
0028  *    Min He <min.he@intel.com>
0029  *    Bing Niu <bing.niu@intel.com>
0030  *    Ping Gao <ping.a.gao@intel.com>
0031  *    Tina Zhang <tina.zhang@intel.com>
0032  *
0033  */
0034 
0035 #include "i915_drv.h"
0036 #include "gvt.h"
0037 
0038 #define _EL_OFFSET_STATUS       0x234
0039 #define _EL_OFFSET_STATUS_BUF   0x370
0040 #define _EL_OFFSET_STATUS_PTR   0x3A0
0041 
0042 #define execlist_ring_mmio(e, offset) ((e)->mmio_base + (offset))
0043 
0044 #define valid_context(ctx) ((ctx)->valid)
0045 #define same_context(a, b) (((a)->context_id == (b)->context_id) && \
0046         ((a)->lrca == (b)->lrca))
0047 
0048 static int context_switch_events[] = {
0049     [RCS0]  = RCS_AS_CONTEXT_SWITCH,
0050     [BCS0]  = BCS_AS_CONTEXT_SWITCH,
0051     [VCS0]  = VCS_AS_CONTEXT_SWITCH,
0052     [VCS1]  = VCS2_AS_CONTEXT_SWITCH,
0053     [VECS0] = VECS_AS_CONTEXT_SWITCH,
0054 };
0055 
0056 static int to_context_switch_event(const struct intel_engine_cs *engine)
0057 {
0058     if (WARN_ON(engine->id >= ARRAY_SIZE(context_switch_events)))
0059         return -EINVAL;
0060 
0061     return context_switch_events[engine->id];
0062 }
0063 
0064 static void switch_virtual_execlist_slot(struct intel_vgpu_execlist *execlist)
0065 {
0066     gvt_dbg_el("[before] running slot %d/context %x pending slot %d\n",
0067             execlist->running_slot ?
0068             execlist->running_slot->index : -1,
0069             execlist->running_context ?
0070             execlist->running_context->context_id : 0,
0071             execlist->pending_slot ?
0072             execlist->pending_slot->index : -1);
0073 
0074     execlist->running_slot = execlist->pending_slot;
0075     execlist->pending_slot = NULL;
0076     execlist->running_context = execlist->running_context ?
0077         &execlist->running_slot->ctx[0] : NULL;
0078 
0079     gvt_dbg_el("[after] running slot %d/context %x pending slot %d\n",
0080             execlist->running_slot ?
0081             execlist->running_slot->index : -1,
0082             execlist->running_context ?
0083             execlist->running_context->context_id : 0,
0084             execlist->pending_slot ?
0085             execlist->pending_slot->index : -1);
0086 }
0087 
0088 static void emulate_execlist_status(struct intel_vgpu_execlist *execlist)
0089 {
0090     struct intel_vgpu_execlist_slot *running = execlist->running_slot;
0091     struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
0092     struct execlist_ctx_descriptor_format *desc = execlist->running_context;
0093     struct intel_vgpu *vgpu = execlist->vgpu;
0094     struct execlist_status_format status;
0095     u32 status_reg =
0096         execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS);
0097 
0098     status.ldw = vgpu_vreg(vgpu, status_reg);
0099     status.udw = vgpu_vreg(vgpu, status_reg + 4);
0100 
0101     if (running) {
0102         status.current_execlist_pointer = !!running->index;
0103         status.execlist_write_pointer = !!!running->index;
0104         status.execlist_0_active = status.execlist_0_valid =
0105             !!!(running->index);
0106         status.execlist_1_active = status.execlist_1_valid =
0107             !!(running->index);
0108     } else {
0109         status.context_id = 0;
0110         status.execlist_0_active = status.execlist_0_valid = 0;
0111         status.execlist_1_active = status.execlist_1_valid = 0;
0112     }
0113 
0114     status.context_id = desc ? desc->context_id : 0;
0115     status.execlist_queue_full = !!(pending);
0116 
0117     vgpu_vreg(vgpu, status_reg) = status.ldw;
0118     vgpu_vreg(vgpu, status_reg + 4) = status.udw;
0119 
0120     gvt_dbg_el("vgpu%d: status reg offset %x ldw %x udw %x\n",
0121         vgpu->id, status_reg, status.ldw, status.udw);
0122 }
0123 
0124 static void emulate_csb_update(struct intel_vgpu_execlist *execlist,
0125                    struct execlist_context_status_format *status,
0126                    bool trigger_interrupt_later)
0127 {
0128     struct intel_vgpu *vgpu = execlist->vgpu;
0129     struct execlist_context_status_pointer_format ctx_status_ptr;
0130     u32 write_pointer;
0131     u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset;
0132     unsigned long hwsp_gpa;
0133 
0134     ctx_status_ptr_reg =
0135         execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS_PTR);
0136     ctx_status_buf_reg =
0137         execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS_BUF);
0138 
0139     ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
0140 
0141     write_pointer = ctx_status_ptr.write_ptr;
0142 
0143     if (write_pointer == 0x7)
0144         write_pointer = 0;
0145     else {
0146         ++write_pointer;
0147         write_pointer %= 0x6;
0148     }
0149 
0150     offset = ctx_status_buf_reg + write_pointer * 8;
0151 
0152     vgpu_vreg(vgpu, offset) = status->ldw;
0153     vgpu_vreg(vgpu, offset + 4) = status->udw;
0154 
0155     ctx_status_ptr.write_ptr = write_pointer;
0156     vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
0157 
0158     /* Update the CSB and CSB write pointer in HWSP */
0159     hwsp_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
0160                      vgpu->hws_pga[execlist->engine->id]);
0161     if (hwsp_gpa != INTEL_GVT_INVALID_ADDR) {
0162         intel_gvt_write_gpa(vgpu,
0163             hwsp_gpa + I915_HWS_CSB_BUF0_INDEX * 4 + write_pointer * 8,
0164             status, 8);
0165         intel_gvt_write_gpa(vgpu,
0166             hwsp_gpa + INTEL_HWS_CSB_WRITE_INDEX(execlist->engine->i915) * 4,
0167             &write_pointer, 4);
0168     }
0169 
0170     gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n",
0171            vgpu->id, write_pointer, offset, status->ldw, status->udw);
0172 
0173     if (trigger_interrupt_later)
0174         return;
0175 
0176     intel_vgpu_trigger_virtual_event(vgpu,
0177                      to_context_switch_event(execlist->engine));
0178 }
0179 
0180 static int emulate_execlist_ctx_schedule_out(
0181         struct intel_vgpu_execlist *execlist,
0182         struct execlist_ctx_descriptor_format *ctx)
0183 {
0184     struct intel_vgpu *vgpu = execlist->vgpu;
0185     struct intel_vgpu_execlist_slot *running = execlist->running_slot;
0186     struct intel_vgpu_execlist_slot *pending = execlist->pending_slot;
0187     struct execlist_ctx_descriptor_format *ctx0 = &running->ctx[0];
0188     struct execlist_ctx_descriptor_format *ctx1 = &running->ctx[1];
0189     struct execlist_context_status_format status;
0190 
0191     memset(&status, 0, sizeof(status));
0192 
0193     gvt_dbg_el("schedule out context id %x\n", ctx->context_id);
0194 
0195     if (WARN_ON(!same_context(ctx, execlist->running_context))) {
0196         gvt_vgpu_err("schedule out context is not running context,"
0197                 "ctx id %x running ctx id %x\n",
0198                 ctx->context_id,
0199                 execlist->running_context->context_id);
0200         return -EINVAL;
0201     }
0202 
0203     /* ctx1 is valid, ctx0/ctx is scheduled-out -> element switch */
0204     if (valid_context(ctx1) && same_context(ctx0, ctx)) {
0205         gvt_dbg_el("ctx 1 valid, ctx/ctx 0 is scheduled-out\n");
0206 
0207         execlist->running_context = ctx1;
0208 
0209         emulate_execlist_status(execlist);
0210 
0211         status.context_complete = status.element_switch = 1;
0212         status.context_id = ctx->context_id;
0213 
0214         emulate_csb_update(execlist, &status, false);
0215         /*
0216          * ctx1 is not valid, ctx == ctx0
0217          * ctx1 is valid, ctx1 == ctx
0218          *  --> last element is finished
0219          * emulate:
0220          *  active-to-idle if there is *no* pending execlist
0221          *  context-complete if there *is* pending execlist
0222          */
0223     } else if ((!valid_context(ctx1) && same_context(ctx0, ctx))
0224             || (valid_context(ctx1) && same_context(ctx1, ctx))) {
0225         gvt_dbg_el("need to switch virtual execlist slot\n");
0226 
0227         switch_virtual_execlist_slot(execlist);
0228 
0229         emulate_execlist_status(execlist);
0230 
0231         status.context_complete = status.active_to_idle = 1;
0232         status.context_id = ctx->context_id;
0233 
0234         if (!pending) {
0235             emulate_csb_update(execlist, &status, false);
0236         } else {
0237             emulate_csb_update(execlist, &status, true);
0238 
0239             memset(&status, 0, sizeof(status));
0240 
0241             status.idle_to_active = 1;
0242             status.context_id = 0;
0243 
0244             emulate_csb_update(execlist, &status, false);
0245         }
0246     } else {
0247         WARN_ON(1);
0248         return -EINVAL;
0249     }
0250 
0251     return 0;
0252 }
0253 
0254 static struct intel_vgpu_execlist_slot *get_next_execlist_slot(
0255         struct intel_vgpu_execlist *execlist)
0256 {
0257     struct intel_vgpu *vgpu = execlist->vgpu;
0258     u32 status_reg =
0259         execlist_ring_mmio(execlist->engine, _EL_OFFSET_STATUS);
0260     struct execlist_status_format status;
0261 
0262     status.ldw = vgpu_vreg(vgpu, status_reg);
0263     status.udw = vgpu_vreg(vgpu, status_reg + 4);
0264 
0265     if (status.execlist_queue_full) {
0266         gvt_vgpu_err("virtual execlist slots are full\n");
0267         return NULL;
0268     }
0269 
0270     return &execlist->slot[status.execlist_write_pointer];
0271 }
0272 
0273 static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist,
0274         struct execlist_ctx_descriptor_format ctx[2])
0275 {
0276     struct intel_vgpu_execlist_slot *running = execlist->running_slot;
0277     struct intel_vgpu_execlist_slot *slot =
0278         get_next_execlist_slot(execlist);
0279 
0280     struct execlist_ctx_descriptor_format *ctx0, *ctx1;
0281     struct execlist_context_status_format status;
0282     struct intel_vgpu *vgpu = execlist->vgpu;
0283 
0284     gvt_dbg_el("emulate schedule-in\n");
0285 
0286     if (!slot) {
0287         gvt_vgpu_err("no available execlist slot\n");
0288         return -EINVAL;
0289     }
0290 
0291     memset(&status, 0, sizeof(status));
0292     memset(slot->ctx, 0, sizeof(slot->ctx));
0293 
0294     slot->ctx[0] = ctx[0];
0295     slot->ctx[1] = ctx[1];
0296 
0297     gvt_dbg_el("alloc slot index %d ctx 0 %x ctx 1 %x\n",
0298             slot->index, ctx[0].context_id,
0299             ctx[1].context_id);
0300 
0301     /*
0302      * no running execlist, make this write bundle as running execlist
0303      * -> idle-to-active
0304      */
0305     if (!running) {
0306         gvt_dbg_el("no current running execlist\n");
0307 
0308         execlist->running_slot = slot;
0309         execlist->pending_slot = NULL;
0310         execlist->running_context = &slot->ctx[0];
0311 
0312         gvt_dbg_el("running slot index %d running context %x\n",
0313                 execlist->running_slot->index,
0314                 execlist->running_context->context_id);
0315 
0316         emulate_execlist_status(execlist);
0317 
0318         status.idle_to_active = 1;
0319         status.context_id = 0;
0320 
0321         emulate_csb_update(execlist, &status, false);
0322         return 0;
0323     }
0324 
0325     ctx0 = &running->ctx[0];
0326     ctx1 = &running->ctx[1];
0327 
0328     gvt_dbg_el("current running slot index %d ctx 0 %x ctx 1 %x\n",
0329         running->index, ctx0->context_id, ctx1->context_id);
0330 
0331     /*
0332      * already has an running execlist
0333      *  a. running ctx1 is valid,
0334      *     ctx0 is finished, and running ctx1 == new execlist ctx[0]
0335      *  b. running ctx1 is not valid,
0336      *     ctx0 == new execlist ctx[0]
0337      * ----> lite-restore + preempted
0338      */
0339     if ((valid_context(ctx1) && same_context(ctx1, &slot->ctx[0]) &&
0340         /* condition a */
0341         (!same_context(ctx0, execlist->running_context))) ||
0342             (!valid_context(ctx1) &&
0343              same_context(ctx0, &slot->ctx[0]))) { /* condition b */
0344         gvt_dbg_el("need to switch virtual execlist slot\n");
0345 
0346         execlist->pending_slot = slot;
0347         switch_virtual_execlist_slot(execlist);
0348 
0349         emulate_execlist_status(execlist);
0350 
0351         status.lite_restore = status.preempted = 1;
0352         status.context_id = ctx[0].context_id;
0353 
0354         emulate_csb_update(execlist, &status, false);
0355     } else {
0356         gvt_dbg_el("emulate as pending slot\n");
0357         /*
0358          * otherwise
0359          * --> emulate pending execlist exist + but no preemption case
0360          */
0361         execlist->pending_slot = slot;
0362         emulate_execlist_status(execlist);
0363     }
0364     return 0;
0365 }
0366 
0367 #define get_desc_from_elsp_dwords(ed, i) \
0368     ((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2]))
0369 
0370 static int prepare_execlist_workload(struct intel_vgpu_workload *workload)
0371 {
0372     struct intel_vgpu *vgpu = workload->vgpu;
0373     struct intel_vgpu_submission *s = &vgpu->submission;
0374     struct execlist_ctx_descriptor_format ctx[2];
0375     int ret;
0376 
0377     if (!workload->emulate_schedule_in)
0378         return 0;
0379 
0380     ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
0381     ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
0382 
0383     ret = emulate_execlist_schedule_in(&s->execlist[workload->engine->id],
0384                        ctx);
0385     if (ret) {
0386         gvt_vgpu_err("fail to emulate execlist schedule in\n");
0387         return ret;
0388     }
0389     return 0;
0390 }
0391 
0392 static int complete_execlist_workload(struct intel_vgpu_workload *workload)
0393 {
0394     struct intel_vgpu *vgpu = workload->vgpu;
0395     struct intel_vgpu_submission *s = &vgpu->submission;
0396     struct intel_vgpu_execlist *execlist =
0397         &s->execlist[workload->engine->id];
0398     struct intel_vgpu_workload *next_workload;
0399     struct list_head *next = workload_q_head(vgpu, workload->engine)->next;
0400     bool lite_restore = false;
0401     int ret = 0;
0402 
0403     gvt_dbg_el("complete workload %p status %d\n",
0404            workload, workload->status);
0405 
0406     if (workload->status || vgpu->resetting_eng & workload->engine->mask)
0407         goto out;
0408 
0409     if (!list_empty(workload_q_head(vgpu, workload->engine))) {
0410         struct execlist_ctx_descriptor_format *this_desc, *next_desc;
0411 
0412         next_workload = container_of(next,
0413                 struct intel_vgpu_workload, list);
0414         this_desc = &workload->ctx_desc;
0415         next_desc = &next_workload->ctx_desc;
0416 
0417         lite_restore = same_context(this_desc, next_desc);
0418     }
0419 
0420     if (lite_restore) {
0421         gvt_dbg_el("next context == current - no schedule-out\n");
0422         goto out;
0423     }
0424 
0425     ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc);
0426 out:
0427     return ret;
0428 }
0429 
0430 static int submit_context(struct intel_vgpu *vgpu,
0431               const struct intel_engine_cs *engine,
0432               struct execlist_ctx_descriptor_format *desc,
0433               bool emulate_schedule_in)
0434 {
0435     struct intel_vgpu_submission *s = &vgpu->submission;
0436     struct intel_vgpu_workload *workload = NULL;
0437 
0438     workload = intel_vgpu_create_workload(vgpu, engine, desc);
0439     if (IS_ERR(workload))
0440         return PTR_ERR(workload);
0441 
0442     workload->prepare = prepare_execlist_workload;
0443     workload->complete = complete_execlist_workload;
0444     workload->emulate_schedule_in = emulate_schedule_in;
0445 
0446     if (emulate_schedule_in)
0447         workload->elsp_dwords = s->execlist[engine->id].elsp_dwords;
0448 
0449     gvt_dbg_el("workload %p emulate schedule_in %d\n", workload,
0450            emulate_schedule_in);
0451 
0452     intel_vgpu_queue_workload(workload);
0453     return 0;
0454 }
0455 
0456 int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu,
0457                    const struct intel_engine_cs *engine)
0458 {
0459     struct intel_vgpu_submission *s = &vgpu->submission;
0460     struct intel_vgpu_execlist *execlist = &s->execlist[engine->id];
0461     struct execlist_ctx_descriptor_format *desc[2];
0462     int i, ret;
0463 
0464     desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
0465     desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
0466 
0467     if (!desc[0]->valid) {
0468         gvt_vgpu_err("invalid elsp submission, desc0 is invalid\n");
0469         goto inv_desc;
0470     }
0471 
0472     for (i = 0; i < ARRAY_SIZE(desc); i++) {
0473         if (!desc[i]->valid)
0474             continue;
0475         if (!desc[i]->privilege_access) {
0476             gvt_vgpu_err("unexpected GGTT elsp submission\n");
0477             goto inv_desc;
0478         }
0479     }
0480 
0481     /* submit workload */
0482     for (i = 0; i < ARRAY_SIZE(desc); i++) {
0483         if (!desc[i]->valid)
0484             continue;
0485         ret = submit_context(vgpu, engine, desc[i], i == 0);
0486         if (ret) {
0487             gvt_vgpu_err("failed to submit desc %d\n", i);
0488             return ret;
0489         }
0490     }
0491 
0492     return 0;
0493 
0494 inv_desc:
0495     gvt_vgpu_err("descriptors content: desc0 %08x %08x desc1 %08x %08x\n",
0496              desc[0]->udw, desc[0]->ldw, desc[1]->udw, desc[1]->ldw);
0497     return -EINVAL;
0498 }
0499 
0500 static void init_vgpu_execlist(struct intel_vgpu *vgpu,
0501                    const struct intel_engine_cs *engine)
0502 {
0503     struct intel_vgpu_submission *s = &vgpu->submission;
0504     struct intel_vgpu_execlist *execlist = &s->execlist[engine->id];
0505     struct execlist_context_status_pointer_format ctx_status_ptr;
0506     u32 ctx_status_ptr_reg;
0507 
0508     memset(execlist, 0, sizeof(*execlist));
0509 
0510     execlist->vgpu = vgpu;
0511     execlist->engine = engine;
0512     execlist->slot[0].index = 0;
0513     execlist->slot[1].index = 1;
0514 
0515     ctx_status_ptr_reg = execlist_ring_mmio(engine, _EL_OFFSET_STATUS_PTR);
0516     ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
0517     ctx_status_ptr.read_ptr = 0;
0518     ctx_status_ptr.write_ptr = 0x7;
0519     vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
0520 }
0521 
0522 static void clean_execlist(struct intel_vgpu *vgpu,
0523                intel_engine_mask_t engine_mask)
0524 {
0525     struct intel_vgpu_submission *s = &vgpu->submission;
0526     struct intel_engine_cs *engine;
0527     intel_engine_mask_t tmp;
0528 
0529     for_each_engine_masked(engine, vgpu->gvt->gt, engine_mask, tmp) {
0530         kfree(s->ring_scan_buffer[engine->id]);
0531         s->ring_scan_buffer[engine->id] = NULL;
0532         s->ring_scan_buffer_size[engine->id] = 0;
0533     }
0534 }
0535 
0536 static void reset_execlist(struct intel_vgpu *vgpu,
0537                intel_engine_mask_t engine_mask)
0538 {
0539     struct intel_engine_cs *engine;
0540     intel_engine_mask_t tmp;
0541 
0542     for_each_engine_masked(engine, vgpu->gvt->gt, engine_mask, tmp)
0543         init_vgpu_execlist(vgpu, engine);
0544 }
0545 
0546 static int init_execlist(struct intel_vgpu *vgpu,
0547              intel_engine_mask_t engine_mask)
0548 {
0549     reset_execlist(vgpu, engine_mask);
0550     return 0;
0551 }
0552 
0553 const struct intel_vgpu_submission_ops intel_vgpu_execlist_submission_ops = {
0554     .name = "execlist",
0555     .init = init_execlist,
0556     .reset = reset_execlist,
0557     .clean = clean_execlist,
0558 };
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!