Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright 2019 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023 
0024 #include <linux/firmware.h>
0025 #include <linux/module.h>
0026 #include "amdgpu.h"
0027 #include "soc15_common.h"
0028 #include "soc21.h"
0029 #include "gc/gc_11_0_0_offset.h"
0030 #include "gc/gc_11_0_0_sh_mask.h"
0031 #include "gc/gc_11_0_0_default.h"
0032 #include "v11_structs.h"
0033 #include "mes_v11_api_def.h"
0034 
0035 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes.bin");
0036 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mes1.bin");
0037 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes.bin");
0038 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes1.bin");
0039 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes.bin");
0040 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes1.bin");
0041 
0042 static int mes_v11_0_hw_fini(void *handle);
0043 static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev);
0044 static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev);
0045 
0046 #define MES_EOP_SIZE   2048
0047 
0048 static void mes_v11_0_ring_set_wptr(struct amdgpu_ring *ring)
0049 {
0050     struct amdgpu_device *adev = ring->adev;
0051 
0052     if (ring->use_doorbell) {
0053         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
0054                  ring->wptr);
0055         WDOORBELL64(ring->doorbell_index, ring->wptr);
0056     } else {
0057         BUG();
0058     }
0059 }
0060 
0061 static u64 mes_v11_0_ring_get_rptr(struct amdgpu_ring *ring)
0062 {
0063     return *ring->rptr_cpu_addr;
0064 }
0065 
0066 static u64 mes_v11_0_ring_get_wptr(struct amdgpu_ring *ring)
0067 {
0068     u64 wptr;
0069 
0070     if (ring->use_doorbell)
0071         wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
0072     else
0073         BUG();
0074     return wptr;
0075 }
0076 
0077 static const struct amdgpu_ring_funcs mes_v11_0_ring_funcs = {
0078     .type = AMDGPU_RING_TYPE_MES,
0079     .align_mask = 1,
0080     .nop = 0,
0081     .support_64bit_ptrs = true,
0082     .get_rptr = mes_v11_0_ring_get_rptr,
0083     .get_wptr = mes_v11_0_ring_get_wptr,
0084     .set_wptr = mes_v11_0_ring_set_wptr,
0085     .insert_nop = amdgpu_ring_insert_nop,
0086 };
0087 
0088 static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
0089                             void *pkt, int size,
0090                             int api_status_off)
0091 {
0092     int ndw = size / 4;
0093     signed long r;
0094     union MESAPI__ADD_QUEUE *x_pkt = pkt;
0095     struct MES_API_STATUS *api_status;
0096     struct amdgpu_device *adev = mes->adev;
0097     struct amdgpu_ring *ring = &mes->ring;
0098     unsigned long flags;
0099 
0100     BUG_ON(size % 4 != 0);
0101 
0102     spin_lock_irqsave(&mes->ring_lock, flags);
0103     if (amdgpu_ring_alloc(ring, ndw)) {
0104         spin_unlock_irqrestore(&mes->ring_lock, flags);
0105         return -ENOMEM;
0106     }
0107 
0108     api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
0109     api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr;
0110     api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq;
0111 
0112     amdgpu_ring_write_multiple(ring, pkt, ndw);
0113     amdgpu_ring_commit(ring);
0114     spin_unlock_irqrestore(&mes->ring_lock, flags);
0115 
0116     DRM_DEBUG("MES msg=%d was emitted\n", x_pkt->header.opcode);
0117 
0118     r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq,
0119               adev->usec_timeout * (amdgpu_emu_mode ? 100 : 1));
0120     if (r < 1) {
0121         DRM_ERROR("MES failed to response msg=%d\n",
0122               x_pkt->header.opcode);
0123         return -ETIMEDOUT;
0124     }
0125 
0126     return 0;
0127 }
0128 
0129 static int convert_to_mes_queue_type(int queue_type)
0130 {
0131     if (queue_type == AMDGPU_RING_TYPE_GFX)
0132         return MES_QUEUE_TYPE_GFX;
0133     else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
0134         return MES_QUEUE_TYPE_COMPUTE;
0135     else if (queue_type == AMDGPU_RING_TYPE_SDMA)
0136         return MES_QUEUE_TYPE_SDMA;
0137     else
0138         BUG();
0139     return -1;
0140 }
0141 
0142 static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,
0143                   struct mes_add_queue_input *input)
0144 {
0145     struct amdgpu_device *adev = mes->adev;
0146     union MESAPI__ADD_QUEUE mes_add_queue_pkt;
0147     struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
0148     uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;
0149 
0150     memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
0151 
0152     mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
0153     mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
0154     mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0155 
0156     mes_add_queue_pkt.process_id = input->process_id;
0157     mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
0158     mes_add_queue_pkt.process_va_start = input->process_va_start;
0159     mes_add_queue_pkt.process_va_end = input->process_va_end;
0160     mes_add_queue_pkt.process_quantum = input->process_quantum;
0161     mes_add_queue_pkt.process_context_addr = input->process_context_addr;
0162     mes_add_queue_pkt.gang_quantum = input->gang_quantum;
0163     mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
0164     mes_add_queue_pkt.inprocess_gang_priority =
0165         input->inprocess_gang_priority;
0166     mes_add_queue_pkt.gang_global_priority_level =
0167         input->gang_global_priority_level;
0168     mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
0169     mes_add_queue_pkt.mqd_addr = input->mqd_addr;
0170 
0171     if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >>
0172             AMDGPU_MES_API_VERSION_SHIFT) >= 2)
0173         mes_add_queue_pkt.wptr_addr = input->wptr_mc_addr;
0174     else
0175         mes_add_queue_pkt.wptr_addr = input->wptr_addr;
0176 
0177     mes_add_queue_pkt.queue_type =
0178         convert_to_mes_queue_type(input->queue_type);
0179     mes_add_queue_pkt.paging = input->paging;
0180     mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;
0181     mes_add_queue_pkt.gws_base = input->gws_base;
0182     mes_add_queue_pkt.gws_size = input->gws_size;
0183     mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
0184     mes_add_queue_pkt.tma_addr = input->tma_addr;
0185     mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
0186     mes_add_queue_pkt.trap_en = 1;
0187 
0188     /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
0189     mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
0190     mes_add_queue_pkt.gds_size = input->queue_size;
0191 
0192     return mes_v11_0_submit_pkt_and_poll_completion(mes,
0193             &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
0194             offsetof(union MESAPI__ADD_QUEUE, api_status));
0195 }
0196 
0197 static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes,
0198                      struct mes_remove_queue_input *input)
0199 {
0200     union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
0201 
0202     memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
0203 
0204     mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
0205     mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
0206     mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0207 
0208     mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
0209     mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
0210 
0211     return mes_v11_0_submit_pkt_and_poll_completion(mes,
0212             &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
0213             offsetof(union MESAPI__REMOVE_QUEUE, api_status));
0214 }
0215 
0216 static int mes_v11_0_unmap_legacy_queue(struct amdgpu_mes *mes,
0217             struct mes_unmap_legacy_queue_input *input)
0218 {
0219     union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
0220 
0221     memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
0222 
0223     mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
0224     mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
0225     mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0226 
0227     mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
0228     mes_remove_queue_pkt.gang_context_addr = 0;
0229 
0230     mes_remove_queue_pkt.pipe_id = input->pipe_id;
0231     mes_remove_queue_pkt.queue_id = input->queue_id;
0232 
0233     if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
0234         mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
0235         mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
0236         mes_remove_queue_pkt.tf_data =
0237             lower_32_bits(input->trail_fence_data);
0238     } else {
0239         mes_remove_queue_pkt.unmap_legacy_queue = 1;
0240         mes_remove_queue_pkt.queue_type =
0241             convert_to_mes_queue_type(input->queue_type);
0242     }
0243 
0244     return mes_v11_0_submit_pkt_and_poll_completion(mes,
0245             &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
0246             offsetof(union MESAPI__REMOVE_QUEUE, api_status));
0247 }
0248 
0249 static int mes_v11_0_suspend_gang(struct amdgpu_mes *mes,
0250                   struct mes_suspend_gang_input *input)
0251 {
0252     return 0;
0253 }
0254 
0255 static int mes_v11_0_resume_gang(struct amdgpu_mes *mes,
0256                  struct mes_resume_gang_input *input)
0257 {
0258     return 0;
0259 }
0260 
0261 static int mes_v11_0_query_sched_status(struct amdgpu_mes *mes)
0262 {
0263     union MESAPI__QUERY_MES_STATUS mes_status_pkt;
0264 
0265     memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
0266 
0267     mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
0268     mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
0269     mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0270 
0271     return mes_v11_0_submit_pkt_and_poll_completion(mes,
0272             &mes_status_pkt, sizeof(mes_status_pkt),
0273             offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
0274 }
0275 
0276 static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
0277                  struct mes_misc_op_input *input)
0278 {
0279     union MESAPI__MISC misc_pkt;
0280 
0281     memset(&misc_pkt, 0, sizeof(misc_pkt));
0282 
0283     misc_pkt.header.type = MES_API_TYPE_SCHEDULER;
0284     misc_pkt.header.opcode = MES_SCH_API_MISC;
0285     misc_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0286 
0287     switch (input->op) {
0288     case MES_MISC_OP_READ_REG:
0289         misc_pkt.opcode = MESAPI_MISC__READ_REG;
0290         misc_pkt.read_reg.reg_offset = input->read_reg.reg_offset;
0291         misc_pkt.read_reg.buffer_addr = input->read_reg.buffer_addr;
0292         break;
0293     case MES_MISC_OP_WRITE_REG:
0294         misc_pkt.opcode = MESAPI_MISC__WRITE_REG;
0295         misc_pkt.write_reg.reg_offset = input->write_reg.reg_offset;
0296         misc_pkt.write_reg.reg_value = input->write_reg.reg_value;
0297         break;
0298     case MES_MISC_OP_WRM_REG_WAIT:
0299         misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
0300         misc_pkt.wait_reg_mem.op = WRM_OPERATION__WAIT_REG_MEM;
0301         misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
0302         misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
0303         misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
0304         misc_pkt.wait_reg_mem.reg_offset2 = 0;
0305         break;
0306     case MES_MISC_OP_WRM_REG_WR_WAIT:
0307         misc_pkt.opcode = MESAPI_MISC__WAIT_REG_MEM;
0308         misc_pkt.wait_reg_mem.op = WRM_OPERATION__WR_WAIT_WR_REG;
0309         misc_pkt.wait_reg_mem.reference = input->wrm_reg.ref;
0310         misc_pkt.wait_reg_mem.mask = input->wrm_reg.mask;
0311         misc_pkt.wait_reg_mem.reg_offset1 = input->wrm_reg.reg0;
0312         misc_pkt.wait_reg_mem.reg_offset2 = input->wrm_reg.reg1;
0313         break;
0314     default:
0315         DRM_ERROR("unsupported misc op (%d) \n", input->op);
0316         return -EINVAL;
0317     }
0318 
0319     return mes_v11_0_submit_pkt_and_poll_completion(mes,
0320             &misc_pkt, sizeof(misc_pkt),
0321             offsetof(union MESAPI__MISC, api_status));
0322 }
0323 
0324 static int mes_v11_0_set_hw_resources(struct amdgpu_mes *mes)
0325 {
0326     int i;
0327     struct amdgpu_device *adev = mes->adev;
0328     union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
0329 
0330     memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
0331 
0332     mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
0333     mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
0334     mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0335 
0336     mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
0337     mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
0338     mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
0339     mes_set_hw_res_pkt.paging_vmid = 0;
0340     mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr;
0341     mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
0342         mes->query_status_fence_gpu_addr;
0343 
0344     for (i = 0; i < MAX_COMPUTE_PIPES; i++)
0345         mes_set_hw_res_pkt.compute_hqd_mask[i] =
0346             mes->compute_hqd_mask[i];
0347 
0348     for (i = 0; i < MAX_GFX_PIPES; i++)
0349         mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i];
0350 
0351     for (i = 0; i < MAX_SDMA_PIPES; i++)
0352         mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];
0353 
0354     for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
0355         mes_set_hw_res_pkt.aggregated_doorbells[i] =
0356             mes->aggregated_doorbells[i];
0357 
0358     for (i = 0; i < 5; i++) {
0359         mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];
0360         mes_set_hw_res_pkt.mmhub_base[i] =
0361                 adev->reg_offset[MMHUB_HWIP][0][i];
0362         mes_set_hw_res_pkt.osssys_base[i] =
0363         adev->reg_offset[OSSSYS_HWIP][0][i];
0364     }
0365 
0366     mes_set_hw_res_pkt.disable_reset = 1;
0367     mes_set_hw_res_pkt.disable_mes_log = 1;
0368     mes_set_hw_res_pkt.use_different_vmid_compute = 1;
0369     mes_set_hw_res_pkt.oversubscription_timer = 50;
0370 
0371     return mes_v11_0_submit_pkt_and_poll_completion(mes,
0372             &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
0373             offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
0374 }
0375 
0376 static void mes_v11_0_init_aggregated_doorbell(struct amdgpu_mes *mes)
0377 {
0378     struct amdgpu_device *adev = mes->adev;
0379     uint32_t data;
0380 
0381     data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1);
0382     data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK |
0383           CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK |
0384           CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK);
0385     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
0386         CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT;
0387     data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT;
0388     WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL1, data);
0389 
0390     data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2);
0391     data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK |
0392           CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK |
0393           CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK);
0394     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
0395         CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT;
0396     data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT;
0397     WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL2, data);
0398 
0399     data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3);
0400     data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK |
0401           CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK |
0402           CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK);
0403     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
0404         CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT;
0405     data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT;
0406     WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL3, data);
0407 
0408     data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4);
0409     data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK |
0410           CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK |
0411           CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK);
0412     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
0413         CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT;
0414     data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT;
0415     WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL4, data);
0416 
0417     data = RREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5);
0418     data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK |
0419           CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK |
0420           CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK);
0421     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
0422         CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT;
0423     data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT;
0424     WREG32_SOC15(GC, 0, regCP_MES_DOORBELL_CONTROL5, data);
0425 
0426     data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT;
0427     WREG32_SOC15(GC, 0, regCP_HQD_GFX_CONTROL, data);
0428 }
0429 
0430 static const struct amdgpu_mes_funcs mes_v11_0_funcs = {
0431     .add_hw_queue = mes_v11_0_add_hw_queue,
0432     .remove_hw_queue = mes_v11_0_remove_hw_queue,
0433     .unmap_legacy_queue = mes_v11_0_unmap_legacy_queue,
0434     .suspend_gang = mes_v11_0_suspend_gang,
0435     .resume_gang = mes_v11_0_resume_gang,
0436     .misc_op = mes_v11_0_misc_op,
0437 };
0438 
0439 static int mes_v11_0_init_microcode(struct amdgpu_device *adev,
0440                     enum admgpu_mes_pipe pipe)
0441 {
0442     char fw_name[30];
0443     char ucode_prefix[30];
0444     int err;
0445     const struct mes_firmware_header_v1_0 *mes_hdr;
0446     struct amdgpu_firmware_info *info;
0447 
0448     amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
0449 
0450     if (pipe == AMDGPU_MES_SCHED_PIPE)
0451         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes.bin",
0452              ucode_prefix);
0453     else
0454         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes1.bin",
0455              ucode_prefix);
0456 
0457     err = request_firmware(&adev->mes.fw[pipe], fw_name, adev->dev);
0458     if (err)
0459         return err;
0460 
0461     err = amdgpu_ucode_validate(adev->mes.fw[pipe]);
0462     if (err) {
0463         release_firmware(adev->mes.fw[pipe]);
0464         adev->mes.fw[pipe] = NULL;
0465         return err;
0466     }
0467 
0468     mes_hdr = (const struct mes_firmware_header_v1_0 *)
0469         adev->mes.fw[pipe]->data;
0470     adev->mes.ucode_fw_version[pipe] =
0471         le32_to_cpu(mes_hdr->mes_ucode_version);
0472     adev->mes.ucode_fw_version[pipe] =
0473         le32_to_cpu(mes_hdr->mes_ucode_data_version);
0474     adev->mes.uc_start_addr[pipe] =
0475         le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) |
0476         ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32);
0477     adev->mes.data_start_addr[pipe] =
0478         le32_to_cpu(mes_hdr->mes_data_start_addr_lo) |
0479         ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32);
0480 
0481     if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
0482         int ucode, ucode_data;
0483 
0484         if (pipe == AMDGPU_MES_SCHED_PIPE) {
0485             ucode = AMDGPU_UCODE_ID_CP_MES;
0486             ucode_data = AMDGPU_UCODE_ID_CP_MES_DATA;
0487         } else {
0488             ucode = AMDGPU_UCODE_ID_CP_MES1;
0489             ucode_data = AMDGPU_UCODE_ID_CP_MES1_DATA;
0490         }
0491 
0492         info = &adev->firmware.ucode[ucode];
0493         info->ucode_id = ucode;
0494         info->fw = adev->mes.fw[pipe];
0495         adev->firmware.fw_size +=
0496             ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes),
0497                   PAGE_SIZE);
0498 
0499         info = &adev->firmware.ucode[ucode_data];
0500         info->ucode_id = ucode_data;
0501         info->fw = adev->mes.fw[pipe];
0502         adev->firmware.fw_size +=
0503             ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes),
0504                   PAGE_SIZE);
0505     }
0506 
0507     return 0;
0508 }
0509 
0510 static void mes_v11_0_free_microcode(struct amdgpu_device *adev,
0511                      enum admgpu_mes_pipe pipe)
0512 {
0513     release_firmware(adev->mes.fw[pipe]);
0514     adev->mes.fw[pipe] = NULL;
0515 }
0516 
0517 static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev,
0518                        enum admgpu_mes_pipe pipe)
0519 {
0520     int r;
0521     const struct mes_firmware_header_v1_0 *mes_hdr;
0522     const __le32 *fw_data;
0523     unsigned fw_size;
0524 
0525     mes_hdr = (const struct mes_firmware_header_v1_0 *)
0526         adev->mes.fw[pipe]->data;
0527 
0528     fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
0529            le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
0530     fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
0531 
0532     r = amdgpu_bo_create_reserved(adev, fw_size,
0533                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
0534                       &adev->mes.ucode_fw_obj[pipe],
0535                       &adev->mes.ucode_fw_gpu_addr[pipe],
0536                       (void **)&adev->mes.ucode_fw_ptr[pipe]);
0537     if (r) {
0538         dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
0539         return r;
0540     }
0541 
0542     memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);
0543 
0544     amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);
0545     amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);
0546 
0547     return 0;
0548 }
0549 
0550 static int mes_v11_0_allocate_ucode_data_buffer(struct amdgpu_device *adev,
0551                         enum admgpu_mes_pipe pipe)
0552 {
0553     int r;
0554     const struct mes_firmware_header_v1_0 *mes_hdr;
0555     const __le32 *fw_data;
0556     unsigned fw_size;
0557 
0558     mes_hdr = (const struct mes_firmware_header_v1_0 *)
0559         adev->mes.fw[pipe]->data;
0560 
0561     fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
0562            le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
0563     fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
0564 
0565     r = amdgpu_bo_create_reserved(adev, fw_size,
0566                       64 * 1024, AMDGPU_GEM_DOMAIN_VRAM,
0567                       &adev->mes.data_fw_obj[pipe],
0568                       &adev->mes.data_fw_gpu_addr[pipe],
0569                       (void **)&adev->mes.data_fw_ptr[pipe]);
0570     if (r) {
0571         dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
0572         return r;
0573     }
0574 
0575     memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);
0576 
0577     amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);
0578     amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);
0579 
0580     return 0;
0581 }
0582 
0583 static void mes_v11_0_free_ucode_buffers(struct amdgpu_device *adev,
0584                      enum admgpu_mes_pipe pipe)
0585 {
0586     amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],
0587                   &adev->mes.data_fw_gpu_addr[pipe],
0588                   (void **)&adev->mes.data_fw_ptr[pipe]);
0589 
0590     amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],
0591                   &adev->mes.ucode_fw_gpu_addr[pipe],
0592                   (void **)&adev->mes.ucode_fw_ptr[pipe]);
0593 }
0594 
0595 static void mes_v11_0_enable(struct amdgpu_device *adev, bool enable)
0596 {
0597     uint64_t ucode_addr;
0598     uint32_t pipe, data = 0;
0599 
0600     if (enable) {
0601         data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
0602         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
0603         data = REG_SET_FIELD(data, CP_MES_CNTL,
0604                  MES_PIPE1_RESET, adev->enable_mes_kiq ? 1 : 0);
0605         WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
0606 
0607         mutex_lock(&adev->srbm_mutex);
0608         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
0609             if (!adev->enable_mes_kiq &&
0610                 pipe == AMDGPU_MES_KIQ_PIPE)
0611                 continue;
0612 
0613             soc21_grbm_select(adev, 3, pipe, 0, 0);
0614 
0615             ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
0616             WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
0617                      lower_32_bits(ucode_addr));
0618             WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
0619                      upper_32_bits(ucode_addr));
0620         }
0621         soc21_grbm_select(adev, 0, 0, 0, 0);
0622         mutex_unlock(&adev->srbm_mutex);
0623 
0624         /* unhalt MES and activate pipe0 */
0625         data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
0626         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE,
0627                      adev->enable_mes_kiq ? 1 : 0);
0628         WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
0629 
0630         if (amdgpu_emu_mode)
0631             msleep(100);
0632         else
0633             udelay(50);
0634     } else {
0635         data = RREG32_SOC15(GC, 0, regCP_MES_CNTL);
0636         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
0637         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
0638         data = REG_SET_FIELD(data, CP_MES_CNTL,
0639                      MES_INVALIDATE_ICACHE, 1);
0640         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
0641         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET,
0642                      adev->enable_mes_kiq ? 1 : 0);
0643         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
0644         WREG32_SOC15(GC, 0, regCP_MES_CNTL, data);
0645     }
0646 }
0647 
0648 /* This function is for backdoor MES firmware */
0649 static int mes_v11_0_load_microcode(struct amdgpu_device *adev,
0650                     enum admgpu_mes_pipe pipe, bool prime_icache)
0651 {
0652     int r;
0653     uint32_t data;
0654     uint64_t ucode_addr;
0655 
0656     mes_v11_0_enable(adev, false);
0657 
0658     if (!adev->mes.fw[pipe])
0659         return -EINVAL;
0660 
0661     r = mes_v11_0_allocate_ucode_buffer(adev, pipe);
0662     if (r)
0663         return r;
0664 
0665     r = mes_v11_0_allocate_ucode_data_buffer(adev, pipe);
0666     if (r) {
0667         mes_v11_0_free_ucode_buffers(adev, pipe);
0668         return r;
0669     }
0670 
0671     mutex_lock(&adev->srbm_mutex);
0672     /* me=3, pipe=0, queue=0 */
0673     soc21_grbm_select(adev, 3, pipe, 0, 0);
0674 
0675     WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0);
0676 
0677     /* set ucode start address */
0678     ucode_addr = adev->mes.uc_start_addr[pipe] >> 2;
0679     WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START,
0680              lower_32_bits(ucode_addr));
0681     WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI,
0682              upper_32_bits(ucode_addr));
0683 
0684     /* set ucode fimrware address */
0685     WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_LO,
0686              lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
0687     WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_HI,
0688              upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
0689 
0690     /* set ucode instruction cache boundary to 2M-1 */
0691     WREG32_SOC15(GC, 0, regCP_MES_MIBOUND_LO, 0x1FFFFF);
0692 
0693     /* set ucode data firmware address */
0694     WREG32_SOC15(GC, 0, regCP_MES_MDBASE_LO,
0695              lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
0696     WREG32_SOC15(GC, 0, regCP_MES_MDBASE_HI,
0697              upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
0698 
0699     /* Set 0x3FFFF (256K-1) to CP_MES_MDBOUND_LO */
0700     WREG32_SOC15(GC, 0, regCP_MES_MDBOUND_LO, 0x3FFFF);
0701 
0702     if (prime_icache) {
0703         /* invalidate ICACHE */
0704         data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
0705         data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
0706         data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
0707         WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
0708 
0709         /* prime the ICACHE. */
0710         data = RREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL);
0711         data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
0712         WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data);
0713     }
0714 
0715     soc21_grbm_select(adev, 0, 0, 0, 0);
0716     mutex_unlock(&adev->srbm_mutex);
0717 
0718     return 0;
0719 }
0720 
0721 static int mes_v11_0_allocate_eop_buf(struct amdgpu_device *adev,
0722                       enum admgpu_mes_pipe pipe)
0723 {
0724     int r;
0725     u32 *eop;
0726 
0727     r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
0728                   AMDGPU_GEM_DOMAIN_GTT,
0729                   &adev->mes.eop_gpu_obj[pipe],
0730                   &adev->mes.eop_gpu_addr[pipe],
0731                   (void **)&eop);
0732     if (r) {
0733         dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
0734         return r;
0735     }
0736 
0737     memset(eop, 0,
0738            adev->mes.eop_gpu_obj[pipe]->tbo.base.size);
0739 
0740     amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);
0741     amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);
0742 
0743     return 0;
0744 }
0745 
0746 static int mes_v11_0_mqd_init(struct amdgpu_ring *ring)
0747 {
0748     struct v11_compute_mqd *mqd = ring->mqd_ptr;
0749     uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
0750     uint32_t tmp;
0751 
0752     mqd->header = 0xC0310800;
0753     mqd->compute_pipelinestat_enable = 0x00000001;
0754     mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
0755     mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
0756     mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
0757     mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
0758     mqd->compute_misc_reserved = 0x00000007;
0759 
0760     eop_base_addr = ring->eop_gpu_addr >> 8;
0761 
0762     /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
0763     tmp = regCP_HQD_EOP_CONTROL_DEFAULT;
0764     tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
0765             (order_base_2(MES_EOP_SIZE / 4) - 1));
0766 
0767     mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
0768     mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
0769     mqd->cp_hqd_eop_control = tmp;
0770 
0771     /* disable the queue if it's active */
0772     ring->wptr = 0;
0773     mqd->cp_hqd_pq_rptr = 0;
0774     mqd->cp_hqd_pq_wptr_lo = 0;
0775     mqd->cp_hqd_pq_wptr_hi = 0;
0776 
0777     /* set the pointer to the MQD */
0778     mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
0779     mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
0780 
0781     /* set MQD vmid to 0 */
0782     tmp = regCP_MQD_CONTROL_DEFAULT;
0783     tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
0784     mqd->cp_mqd_control = tmp;
0785 
0786     /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
0787     hqd_gpu_addr = ring->gpu_addr >> 8;
0788     mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
0789     mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
0790 
0791     /* set the wb address whether it's enabled or not */
0792     wb_gpu_addr = ring->rptr_gpu_addr;
0793     mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
0794     mqd->cp_hqd_pq_rptr_report_addr_hi =
0795         upper_32_bits(wb_gpu_addr) & 0xffff;
0796 
0797     /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
0798     wb_gpu_addr = ring->wptr_gpu_addr;
0799     mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
0800     mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
0801 
0802     /* set up the HQD, this is similar to CP_RB0_CNTL */
0803     tmp = regCP_HQD_PQ_CONTROL_DEFAULT;
0804     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
0805                 (order_base_2(ring->ring_size / 4) - 1));
0806     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
0807                 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
0808     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
0809     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
0810     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
0811     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
0812     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
0813     mqd->cp_hqd_pq_control = tmp;
0814 
0815     /* enable doorbell */
0816     tmp = 0;
0817     if (ring->use_doorbell) {
0818         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0819                     DOORBELL_OFFSET, ring->doorbell_index);
0820         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0821                     DOORBELL_EN, 1);
0822         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0823                     DOORBELL_SOURCE, 0);
0824         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0825                     DOORBELL_HIT, 0);
0826     }
0827     else
0828         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0829                     DOORBELL_EN, 0);
0830     mqd->cp_hqd_pq_doorbell_control = tmp;
0831 
0832     mqd->cp_hqd_vmid = 0;
0833     /* activate the queue */
0834     mqd->cp_hqd_active = 1;
0835 
0836     tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT;
0837     tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE,
0838                 PRELOAD_SIZE, 0x55);
0839     mqd->cp_hqd_persistent_state = tmp;
0840 
0841     mqd->cp_hqd_ib_control = regCP_HQD_IB_CONTROL_DEFAULT;
0842     mqd->cp_hqd_iq_timer = regCP_HQD_IQ_TIMER_DEFAULT;
0843     mqd->cp_hqd_quantum = regCP_HQD_QUANTUM_DEFAULT;
0844 
0845     return 0;
0846 }
0847 
0848 static void mes_v11_0_queue_init_register(struct amdgpu_ring *ring)
0849 {
0850     struct v11_compute_mqd *mqd = ring->mqd_ptr;
0851     struct amdgpu_device *adev = ring->adev;
0852     uint32_t data = 0;
0853 
0854     mutex_lock(&adev->srbm_mutex);
0855     soc21_grbm_select(adev, 3, ring->pipe, 0, 0);
0856 
0857     /* set CP_HQD_VMID.VMID = 0. */
0858     data = RREG32_SOC15(GC, 0, regCP_HQD_VMID);
0859     data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
0860     WREG32_SOC15(GC, 0, regCP_HQD_VMID, data);
0861 
0862     /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
0863     data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL);
0864     data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
0865                  DOORBELL_EN, 0);
0866     WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data);
0867 
0868     /* set CP_MQD_BASE_ADDR/HI with the MQD base address */
0869     WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
0870     WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
0871 
0872     /* set CP_MQD_CONTROL.VMID=0 */
0873     data = RREG32_SOC15(GC, 0, regCP_MQD_CONTROL);
0874     data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
0875     WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 0);
0876 
0877     /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
0878     WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
0879     WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
0880 
0881     /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
0882     WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR,
0883              mqd->cp_hqd_pq_rptr_report_addr_lo);
0884     WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
0885              mqd->cp_hqd_pq_rptr_report_addr_hi);
0886 
0887     /* set CP_HQD_PQ_CONTROL */
0888     WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
0889 
0890     /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
0891     WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR,
0892              mqd->cp_hqd_pq_wptr_poll_addr_lo);
0893     WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI,
0894              mqd->cp_hqd_pq_wptr_poll_addr_hi);
0895 
0896     /* set CP_HQD_PQ_DOORBELL_CONTROL */
0897     WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL,
0898              mqd->cp_hqd_pq_doorbell_control);
0899 
0900     /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
0901     WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
0902 
0903     /* set CP_HQD_ACTIVE.ACTIVE=1 */
0904     WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active);
0905 
0906     soc21_grbm_select(adev, 0, 0, 0, 0);
0907     mutex_unlock(&adev->srbm_mutex);
0908 }
0909 
0910 static int mes_v11_0_kiq_enable_queue(struct amdgpu_device *adev)
0911 {
0912     struct amdgpu_kiq *kiq = &adev->gfx.kiq;
0913     struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
0914     int r;
0915 
0916     if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
0917         return -EINVAL;
0918 
0919     r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
0920     if (r) {
0921         DRM_ERROR("Failed to lock KIQ (%d).\n", r);
0922         return r;
0923     }
0924 
0925     kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring);
0926 
0927     r = amdgpu_ring_test_ring(kiq_ring);
0928     if (r) {
0929         DRM_ERROR("kfq enable failed\n");
0930         kiq_ring->sched.ready = false;
0931     }
0932     return r;
0933 }
0934 
0935 static int mes_v11_0_queue_init(struct amdgpu_device *adev,
0936                 enum admgpu_mes_pipe pipe)
0937 {
0938     struct amdgpu_ring *ring;
0939     int r;
0940 
0941     if (pipe == AMDGPU_MES_KIQ_PIPE)
0942         ring = &adev->gfx.kiq.ring;
0943     else if (pipe == AMDGPU_MES_SCHED_PIPE)
0944         ring = &adev->mes.ring;
0945     else
0946         BUG();
0947 
0948     if ((pipe == AMDGPU_MES_SCHED_PIPE) &&
0949         (amdgpu_in_reset(adev) || adev->in_suspend)) {
0950         *(ring->wptr_cpu_addr) = 0;
0951         *(ring->rptr_cpu_addr) = 0;
0952         amdgpu_ring_clear_ring(ring);
0953     }
0954 
0955     r = mes_v11_0_mqd_init(ring);
0956     if (r)
0957         return r;
0958 
0959     if (pipe == AMDGPU_MES_SCHED_PIPE) {
0960         r = mes_v11_0_kiq_enable_queue(adev);
0961         if (r)
0962             return r;
0963     } else {
0964         mes_v11_0_queue_init_register(ring);
0965     }
0966 
0967     /* get MES scheduler/KIQ versions */
0968     mutex_lock(&adev->srbm_mutex);
0969     soc21_grbm_select(adev, 3, pipe, 0, 0);
0970 
0971     if (pipe == AMDGPU_MES_SCHED_PIPE)
0972         adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
0973     else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
0974         adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO);
0975 
0976     soc21_grbm_select(adev, 0, 0, 0, 0);
0977     mutex_unlock(&adev->srbm_mutex);
0978 
0979     return 0;
0980 }
0981 
0982 static int mes_v11_0_ring_init(struct amdgpu_device *adev)
0983 {
0984     struct amdgpu_ring *ring;
0985 
0986     ring = &adev->mes.ring;
0987 
0988     ring->funcs = &mes_v11_0_ring_funcs;
0989 
0990     ring->me = 3;
0991     ring->pipe = 0;
0992     ring->queue = 0;
0993 
0994     ring->ring_obj = NULL;
0995     ring->use_doorbell = true;
0996     ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;
0997     ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE];
0998     ring->no_scheduler = true;
0999     sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1000 
1001     return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1002                 AMDGPU_RING_PRIO_DEFAULT, NULL);
1003 }
1004 
1005 static int mes_v11_0_kiq_ring_init(struct amdgpu_device *adev)
1006 {
1007     struct amdgpu_ring *ring;
1008 
1009     spin_lock_init(&adev->gfx.kiq.ring_lock);
1010 
1011     ring = &adev->gfx.kiq.ring;
1012 
1013     ring->me = 3;
1014     ring->pipe = 1;
1015     ring->queue = 0;
1016 
1017     ring->adev = NULL;
1018     ring->ring_obj = NULL;
1019     ring->use_doorbell = true;
1020     ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
1021     ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];
1022     ring->no_scheduler = true;
1023     sprintf(ring->name, "mes_kiq_%d.%d.%d",
1024         ring->me, ring->pipe, ring->queue);
1025 
1026     return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
1027                 AMDGPU_RING_PRIO_DEFAULT, NULL);
1028 }
1029 
1030 static int mes_v11_0_mqd_sw_init(struct amdgpu_device *adev,
1031                  enum admgpu_mes_pipe pipe)
1032 {
1033     int r, mqd_size = sizeof(struct v11_compute_mqd);
1034     struct amdgpu_ring *ring;
1035 
1036     if (pipe == AMDGPU_MES_KIQ_PIPE)
1037         ring = &adev->gfx.kiq.ring;
1038     else if (pipe == AMDGPU_MES_SCHED_PIPE)
1039         ring = &adev->mes.ring;
1040     else
1041         BUG();
1042 
1043     if (ring->mqd_obj)
1044         return 0;
1045 
1046     r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
1047                     AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
1048                     &ring->mqd_gpu_addr, &ring->mqd_ptr);
1049     if (r) {
1050         dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
1051         return r;
1052     }
1053 
1054     memset(ring->mqd_ptr, 0, mqd_size);
1055 
1056     /* prepare MQD backup */
1057     adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);
1058     if (!adev->mes.mqd_backup[pipe])
1059         dev_warn(adev->dev,
1060              "no memory to create MQD backup for ring %s\n",
1061              ring->name);
1062 
1063     return 0;
1064 }
1065 
1066 static int mes_v11_0_sw_init(void *handle)
1067 {
1068     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1069     int pipe, r;
1070 
1071     adev->mes.adev = adev;
1072     adev->mes.funcs = &mes_v11_0_funcs;
1073     adev->mes.kiq_hw_init = &mes_v11_0_kiq_hw_init;
1074     adev->mes.kiq_hw_fini = &mes_v11_0_kiq_hw_fini;
1075 
1076     r = amdgpu_mes_init(adev);
1077     if (r)
1078         return r;
1079 
1080     for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1081         if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)
1082             continue;
1083 
1084         r = mes_v11_0_init_microcode(adev, pipe);
1085         if (r)
1086             return r;
1087 
1088         r = mes_v11_0_allocate_eop_buf(adev, pipe);
1089         if (r)
1090             return r;
1091 
1092         r = mes_v11_0_mqd_sw_init(adev, pipe);
1093         if (r)
1094             return r;
1095     }
1096 
1097     if (adev->enable_mes_kiq) {
1098         r = mes_v11_0_kiq_ring_init(adev);
1099         if (r)
1100             return r;
1101     }
1102 
1103     r = mes_v11_0_ring_init(adev);
1104     if (r)
1105         return r;
1106 
1107     return 0;
1108 }
1109 
1110 static int mes_v11_0_sw_fini(void *handle)
1111 {
1112     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1113     int pipe;
1114 
1115     amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
1116     amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
1117 
1118     for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1119         kfree(adev->mes.mqd_backup[pipe]);
1120 
1121         amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],
1122                       &adev->mes.eop_gpu_addr[pipe],
1123                       NULL);
1124 
1125         mes_v11_0_free_microcode(adev, pipe);
1126     }
1127 
1128     amdgpu_bo_free_kernel(&adev->gfx.kiq.ring.mqd_obj,
1129                   &adev->gfx.kiq.ring.mqd_gpu_addr,
1130                   &adev->gfx.kiq.ring.mqd_ptr);
1131 
1132     amdgpu_bo_free_kernel(&adev->mes.ring.mqd_obj,
1133                   &adev->mes.ring.mqd_gpu_addr,
1134                   &adev->mes.ring.mqd_ptr);
1135 
1136     amdgpu_ring_fini(&adev->gfx.kiq.ring);
1137     amdgpu_ring_fini(&adev->mes.ring);
1138 
1139     if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1140         mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);
1141         mes_v11_0_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);
1142     }
1143 
1144     amdgpu_mes_fini(adev);
1145     return 0;
1146 }
1147 
1148 static void mes_v11_0_kiq_setting(struct amdgpu_ring *ring)
1149 {
1150     uint32_t tmp;
1151     struct amdgpu_device *adev = ring->adev;
1152 
1153     /* tell RLC which is KIQ queue */
1154     tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS);
1155     tmp &= 0xffffff00;
1156     tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1157     WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
1158     tmp |= 0x80;
1159     WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp);
1160 }
1161 
1162 static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev)
1163 {
1164     int r = 0;
1165 
1166     if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1167 
1168         r = mes_v11_0_load_microcode(adev, AMDGPU_MES_SCHED_PIPE, false);
1169         if (r) {
1170             DRM_ERROR("failed to load MES fw, r=%d\n", r);
1171             return r;
1172         }
1173 
1174         r = mes_v11_0_load_microcode(adev, AMDGPU_MES_KIQ_PIPE, true);
1175         if (r) {
1176             DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
1177             return r;
1178         }
1179 
1180     }
1181 
1182     mes_v11_0_enable(adev, true);
1183 
1184     mes_v11_0_kiq_setting(&adev->gfx.kiq.ring);
1185 
1186     r = mes_v11_0_queue_init(adev, AMDGPU_MES_KIQ_PIPE);
1187     if (r)
1188         goto failure;
1189 
1190     return r;
1191 
1192 failure:
1193     mes_v11_0_hw_fini(adev);
1194     return r;
1195 }
1196 
1197 static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev)
1198 {
1199     mes_v11_0_enable(adev, false);
1200     return 0;
1201 }
1202 
1203 static int mes_v11_0_hw_init(void *handle)
1204 {
1205     int r;
1206     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1207 
1208     if (!adev->enable_mes_kiq) {
1209         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1210             r = mes_v11_0_load_microcode(adev,
1211                          AMDGPU_MES_SCHED_PIPE, true);
1212             if (r) {
1213                 DRM_ERROR("failed to MES fw, r=%d\n", r);
1214                 return r;
1215             }
1216         }
1217 
1218         mes_v11_0_enable(adev, true);
1219     }
1220 
1221     r = mes_v11_0_queue_init(adev, AMDGPU_MES_SCHED_PIPE);
1222     if (r)
1223         goto failure;
1224 
1225     r = mes_v11_0_set_hw_resources(&adev->mes);
1226     if (r)
1227         goto failure;
1228 
1229     mes_v11_0_init_aggregated_doorbell(&adev->mes);
1230 
1231     r = mes_v11_0_query_sched_status(&adev->mes);
1232     if (r) {
1233         DRM_ERROR("MES is busy\n");
1234         goto failure;
1235     }
1236 
1237     /*
1238      * Disable KIQ ring usage from the driver once MES is enabled.
1239      * MES uses KIQ ring exclusively so driver cannot access KIQ ring
1240      * with MES enabled.
1241      */
1242     adev->gfx.kiq.ring.sched.ready = false;
1243     adev->mes.ring.sched.ready = true;
1244 
1245     return 0;
1246 
1247 failure:
1248     mes_v11_0_hw_fini(adev);
1249     return r;
1250 }
1251 
1252 static int mes_v11_0_hw_fini(void *handle)
1253 {
1254     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1255 
1256     adev->mes.ring.sched.ready = false;
1257     return 0;
1258 }
1259 
1260 static int mes_v11_0_suspend(void *handle)
1261 {
1262     int r;
1263     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1264 
1265     r = amdgpu_mes_suspend(adev);
1266     if (r)
1267         return r;
1268 
1269     return mes_v11_0_hw_fini(adev);
1270 }
1271 
1272 static int mes_v11_0_resume(void *handle)
1273 {
1274     int r;
1275     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1276 
1277     r = mes_v11_0_hw_init(adev);
1278     if (r)
1279         return r;
1280 
1281     return amdgpu_mes_resume(adev);
1282 }
1283 
1284 static int mes_v11_0_late_init(void *handle)
1285 {
1286     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1287 
1288     if (!amdgpu_in_reset(adev))
1289         amdgpu_mes_self_test(adev);
1290 
1291     return 0;
1292 }
1293 
1294 static const struct amd_ip_funcs mes_v11_0_ip_funcs = {
1295     .name = "mes_v11_0",
1296     .late_init = mes_v11_0_late_init,
1297     .sw_init = mes_v11_0_sw_init,
1298     .sw_fini = mes_v11_0_sw_fini,
1299     .hw_init = mes_v11_0_hw_init,
1300     .hw_fini = mes_v11_0_hw_fini,
1301     .suspend = mes_v11_0_suspend,
1302     .resume = mes_v11_0_resume,
1303 };
1304 
1305 const struct amdgpu_ip_block_version mes_v11_0_ip_block = {
1306     .type = AMD_IP_BLOCK_TYPE_MES,
1307     .major = 11,
1308     .minor = 0,
1309     .rev = 0,
1310     .funcs = &mes_v11_0_ip_funcs,
1311 };
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!