Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright 2019 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023 
0024 #include <linux/firmware.h>
0025 #include <linux/module.h>
0026 #include "amdgpu.h"
0027 #include "soc15_common.h"
0028 #include "nv.h"
0029 #include "gc/gc_10_1_0_offset.h"
0030 #include "gc/gc_10_1_0_sh_mask.h"
0031 #include "gc/gc_10_1_0_default.h"
0032 #include "v10_structs.h"
0033 #include "mes_api_def.h"
0034 
0035 #define mmCP_MES_IC_OP_CNTL_Sienna_Cichlid               0x2820
0036 #define mmCP_MES_IC_OP_CNTL_Sienna_Cichlid_BASE_IDX      1
0037 #define mmRLC_CP_SCHEDULERS_Sienna_Cichlid      0x4ca1
0038 #define mmRLC_CP_SCHEDULERS_Sienna_Cichlid_BASE_IDX 1
0039 
0040 MODULE_FIRMWARE("amdgpu/navi10_mes.bin");
0041 MODULE_FIRMWARE("amdgpu/sienna_cichlid_mes.bin");
0042 MODULE_FIRMWARE("amdgpu/sienna_cichlid_mes1.bin");
0043 
0044 static int mes_v10_1_hw_fini(void *handle);
0045 static int mes_v10_1_kiq_hw_init(struct amdgpu_device *adev);
0046 
0047 #define MES_EOP_SIZE   2048
0048 
0049 static void mes_v10_1_ring_set_wptr(struct amdgpu_ring *ring)
0050 {
0051     struct amdgpu_device *adev = ring->adev;
0052 
0053     if (ring->use_doorbell) {
0054         atomic64_set((atomic64_t *)ring->wptr_cpu_addr,
0055                  ring->wptr);
0056         WDOORBELL64(ring->doorbell_index, ring->wptr);
0057     } else {
0058         BUG();
0059     }
0060 }
0061 
0062 static u64 mes_v10_1_ring_get_rptr(struct amdgpu_ring *ring)
0063 {
0064     return *ring->rptr_cpu_addr;
0065 }
0066 
0067 static u64 mes_v10_1_ring_get_wptr(struct amdgpu_ring *ring)
0068 {
0069     u64 wptr;
0070 
0071     if (ring->use_doorbell)
0072         wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr);
0073     else
0074         BUG();
0075     return wptr;
0076 }
0077 
0078 static const struct amdgpu_ring_funcs mes_v10_1_ring_funcs = {
0079     .type = AMDGPU_RING_TYPE_MES,
0080     .align_mask = 1,
0081     .nop = 0,
0082     .support_64bit_ptrs = true,
0083     .get_rptr = mes_v10_1_ring_get_rptr,
0084     .get_wptr = mes_v10_1_ring_get_wptr,
0085     .set_wptr = mes_v10_1_ring_set_wptr,
0086     .insert_nop = amdgpu_ring_insert_nop,
0087 };
0088 
0089 static int mes_v10_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
0090                             void *pkt, int size,
0091                             int api_status_off)
0092 {
0093     int ndw = size / 4;
0094     signed long r;
0095     union MESAPI__ADD_QUEUE *x_pkt = pkt;
0096     struct MES_API_STATUS *api_status;
0097     struct amdgpu_device *adev = mes->adev;
0098     struct amdgpu_ring *ring = &mes->ring;
0099     unsigned long flags;
0100 
0101     BUG_ON(size % 4 != 0);
0102 
0103     spin_lock_irqsave(&mes->ring_lock, flags);
0104     if (amdgpu_ring_alloc(ring, ndw)) {
0105         spin_unlock_irqrestore(&mes->ring_lock, flags);
0106         return -ENOMEM;
0107     }
0108 
0109     api_status = (struct MES_API_STATUS *)((char *)pkt + api_status_off);
0110     api_status->api_completion_fence_addr = mes->ring.fence_drv.gpu_addr;
0111     api_status->api_completion_fence_value = ++mes->ring.fence_drv.sync_seq;
0112 
0113     amdgpu_ring_write_multiple(ring, pkt, ndw);
0114     amdgpu_ring_commit(ring);
0115     spin_unlock_irqrestore(&mes->ring_lock, flags);
0116 
0117     DRM_DEBUG("MES msg=%d was emitted\n", x_pkt->header.opcode);
0118 
0119     r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq,
0120                       adev->usec_timeout);
0121     if (r < 1) {
0122         DRM_ERROR("MES failed to response msg=%d\n",
0123               x_pkt->header.opcode);
0124         return -ETIMEDOUT;
0125     }
0126 
0127     return 0;
0128 }
0129 
0130 static int convert_to_mes_queue_type(int queue_type)
0131 {
0132     if (queue_type == AMDGPU_RING_TYPE_GFX)
0133         return MES_QUEUE_TYPE_GFX;
0134     else if (queue_type == AMDGPU_RING_TYPE_COMPUTE)
0135         return MES_QUEUE_TYPE_COMPUTE;
0136     else if (queue_type == AMDGPU_RING_TYPE_SDMA)
0137         return MES_QUEUE_TYPE_SDMA;
0138     else
0139         BUG();
0140     return -1;
0141 }
0142 
0143 static int mes_v10_1_add_hw_queue(struct amdgpu_mes *mes,
0144                   struct mes_add_queue_input *input)
0145 {
0146     struct amdgpu_device *adev = mes->adev;
0147     union MESAPI__ADD_QUEUE mes_add_queue_pkt;
0148     struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
0149     uint32_t vm_cntx_cntl = hub->vm_cntx_cntl;
0150 
0151     memset(&mes_add_queue_pkt, 0, sizeof(mes_add_queue_pkt));
0152 
0153     mes_add_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
0154     mes_add_queue_pkt.header.opcode = MES_SCH_API_ADD_QUEUE;
0155     mes_add_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0156 
0157     mes_add_queue_pkt.process_id = input->process_id;
0158     mes_add_queue_pkt.page_table_base_addr = input->page_table_base_addr;
0159     mes_add_queue_pkt.process_va_start = input->process_va_start;
0160     mes_add_queue_pkt.process_va_end = input->process_va_end;
0161     mes_add_queue_pkt.process_quantum = input->process_quantum;
0162     mes_add_queue_pkt.process_context_addr = input->process_context_addr;
0163     mes_add_queue_pkt.gang_quantum = input->gang_quantum;
0164     mes_add_queue_pkt.gang_context_addr = input->gang_context_addr;
0165     mes_add_queue_pkt.inprocess_gang_priority =
0166         input->inprocess_gang_priority;
0167     mes_add_queue_pkt.gang_global_priority_level =
0168         input->gang_global_priority_level;
0169     mes_add_queue_pkt.doorbell_offset = input->doorbell_offset;
0170     mes_add_queue_pkt.mqd_addr = input->mqd_addr;
0171     mes_add_queue_pkt.wptr_addr = input->wptr_addr;
0172     mes_add_queue_pkt.queue_type =
0173         convert_to_mes_queue_type(input->queue_type);
0174     mes_add_queue_pkt.paging = input->paging;
0175     mes_add_queue_pkt.vm_context_cntl = vm_cntx_cntl;
0176     mes_add_queue_pkt.gws_base = input->gws_base;
0177     mes_add_queue_pkt.gws_size = input->gws_size;
0178     mes_add_queue_pkt.trap_handler_addr = input->tba_addr;
0179 
0180     return mes_v10_1_submit_pkt_and_poll_completion(mes,
0181             &mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
0182             offsetof(union MESAPI__ADD_QUEUE, api_status));
0183 }
0184 
0185 static int mes_v10_1_remove_hw_queue(struct amdgpu_mes *mes,
0186                      struct mes_remove_queue_input *input)
0187 {
0188     union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
0189 
0190     memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
0191 
0192     mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
0193     mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
0194     mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0195 
0196     mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
0197     mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr;
0198 
0199     return mes_v10_1_submit_pkt_and_poll_completion(mes,
0200             &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
0201             offsetof(union MESAPI__REMOVE_QUEUE, api_status));
0202 }
0203 
0204 static int mes_v10_1_unmap_legacy_queue(struct amdgpu_mes *mes,
0205                  struct mes_unmap_legacy_queue_input *input)
0206 {
0207     union MESAPI__REMOVE_QUEUE mes_remove_queue_pkt;
0208 
0209     memset(&mes_remove_queue_pkt, 0, sizeof(mes_remove_queue_pkt));
0210 
0211     mes_remove_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
0212     mes_remove_queue_pkt.header.opcode = MES_SCH_API_REMOVE_QUEUE;
0213     mes_remove_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0214 
0215     mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset;
0216     mes_remove_queue_pkt.gang_context_addr = 0;
0217 
0218     mes_remove_queue_pkt.pipe_id = input->pipe_id;
0219     mes_remove_queue_pkt.queue_id = input->queue_id;
0220 
0221     if (input->action == PREEMPT_QUEUES_NO_UNMAP) {
0222         mes_remove_queue_pkt.preempt_legacy_gfx_queue = 1;
0223         mes_remove_queue_pkt.tf_addr = input->trail_fence_addr;
0224         mes_remove_queue_pkt.tf_data =
0225             lower_32_bits(input->trail_fence_data);
0226     } else {
0227         if (input->queue_type == AMDGPU_RING_TYPE_GFX)
0228             mes_remove_queue_pkt.unmap_legacy_gfx_queue = 1;
0229         else
0230             mes_remove_queue_pkt.unmap_kiq_utility_queue = 1;
0231     }
0232 
0233     return mes_v10_1_submit_pkt_and_poll_completion(mes,
0234             &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt),
0235             offsetof(union MESAPI__REMOVE_QUEUE, api_status));
0236 }
0237 
0238 static int mes_v10_1_suspend_gang(struct amdgpu_mes *mes,
0239                   struct mes_suspend_gang_input *input)
0240 {
0241     return 0;
0242 }
0243 
0244 static int mes_v10_1_resume_gang(struct amdgpu_mes *mes,
0245                  struct mes_resume_gang_input *input)
0246 {
0247     return 0;
0248 }
0249 
0250 static int mes_v10_1_query_sched_status(struct amdgpu_mes *mes)
0251 {
0252     union MESAPI__QUERY_MES_STATUS mes_status_pkt;
0253 
0254     memset(&mes_status_pkt, 0, sizeof(mes_status_pkt));
0255 
0256     mes_status_pkt.header.type = MES_API_TYPE_SCHEDULER;
0257     mes_status_pkt.header.opcode = MES_SCH_API_QUERY_SCHEDULER_STATUS;
0258     mes_status_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0259 
0260     return mes_v10_1_submit_pkt_and_poll_completion(mes,
0261             &mes_status_pkt, sizeof(mes_status_pkt),
0262             offsetof(union MESAPI__QUERY_MES_STATUS, api_status));
0263 }
0264 
0265 static int mes_v10_1_set_hw_resources(struct amdgpu_mes *mes)
0266 {
0267     int i;
0268     struct amdgpu_device *adev = mes->adev;
0269     union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
0270 
0271     memset(&mes_set_hw_res_pkt, 0, sizeof(mes_set_hw_res_pkt));
0272 
0273     mes_set_hw_res_pkt.header.type = MES_API_TYPE_SCHEDULER;
0274     mes_set_hw_res_pkt.header.opcode = MES_SCH_API_SET_HW_RSRC;
0275     mes_set_hw_res_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
0276 
0277     mes_set_hw_res_pkt.vmid_mask_mmhub = mes->vmid_mask_mmhub;
0278     mes_set_hw_res_pkt.vmid_mask_gfxhub = mes->vmid_mask_gfxhub;
0279     mes_set_hw_res_pkt.gds_size = adev->gds.gds_size;
0280     mes_set_hw_res_pkt.paging_vmid = 0;
0281     mes_set_hw_res_pkt.g_sch_ctx_gpu_mc_ptr = mes->sch_ctx_gpu_addr;
0282     mes_set_hw_res_pkt.query_status_fence_gpu_mc_ptr =
0283         mes->query_status_fence_gpu_addr;
0284 
0285     for (i = 0; i < MAX_COMPUTE_PIPES; i++)
0286         mes_set_hw_res_pkt.compute_hqd_mask[i] =
0287             mes->compute_hqd_mask[i];
0288 
0289     for (i = 0; i < MAX_GFX_PIPES; i++)
0290         mes_set_hw_res_pkt.gfx_hqd_mask[i] = mes->gfx_hqd_mask[i];
0291 
0292     for (i = 0; i < MAX_SDMA_PIPES; i++)
0293         mes_set_hw_res_pkt.sdma_hqd_mask[i] = mes->sdma_hqd_mask[i];
0294 
0295     for (i = 0; i < AMD_PRIORITY_NUM_LEVELS; i++)
0296         mes_set_hw_res_pkt.aggregated_doorbells[i] =
0297             mes->aggregated_doorbells[i];
0298 
0299     for (i = 0; i < 5; i++) {
0300         mes_set_hw_res_pkt.gc_base[i] = adev->reg_offset[GC_HWIP][0][i];
0301         mes_set_hw_res_pkt.mmhub_base[i] =
0302             adev->reg_offset[MMHUB_HWIP][0][i];
0303         mes_set_hw_res_pkt.osssys_base[i] =
0304             adev->reg_offset[OSSSYS_HWIP][0][i];
0305     }
0306 
0307     mes_set_hw_res_pkt.disable_reset = 1;
0308     mes_set_hw_res_pkt.disable_mes_log = 1;
0309     mes_set_hw_res_pkt.use_different_vmid_compute = 1;
0310 
0311     return mes_v10_1_submit_pkt_and_poll_completion(mes,
0312             &mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
0313             offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
0314 }
0315 
0316 static void mes_v10_1_init_aggregated_doorbell(struct amdgpu_mes *mes)
0317 {
0318     struct amdgpu_device *adev = mes->adev;
0319     uint32_t data;
0320 
0321     data = RREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL1);
0322     data &= ~(CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK |
0323           CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK |
0324           CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK);
0325     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_LOW] <<
0326         CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT;
0327     data |= 1 << CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT;
0328     WREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL1, data);
0329 
0330     data = RREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL2);
0331     data &= ~(CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK |
0332           CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK |
0333           CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK);
0334     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] <<
0335         CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT;
0336     data |= 1 << CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT;
0337     WREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL2, data);
0338 
0339     data = RREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL3);
0340     data &= ~(CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK |
0341           CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK |
0342           CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK);
0343     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_MEDIUM] <<
0344         CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT;
0345     data |= 1 << CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT;
0346     WREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL3, data);
0347 
0348     data = RREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL4);
0349     data &= ~(CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK |
0350           CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK |
0351           CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK);
0352     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_HIGH] <<
0353         CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT;
0354     data |= 1 << CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT;
0355     WREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL4, data);
0356 
0357     data = RREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL5);
0358     data &= ~(CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK |
0359           CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK |
0360           CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK);
0361     data |= mes->aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_REALTIME] <<
0362         CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT;
0363     data |= 1 << CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT;
0364     WREG32_SOC15(GC, 0, mmCP_MES_DOORBELL_CONTROL5, data);
0365 
0366     data = 1 << CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT;
0367     WREG32_SOC15(GC, 0, mmCP_HQD_GFX_CONTROL, data);
0368 }
0369 
0370 static const struct amdgpu_mes_funcs mes_v10_1_funcs = {
0371     .add_hw_queue = mes_v10_1_add_hw_queue,
0372     .remove_hw_queue = mes_v10_1_remove_hw_queue,
0373     .unmap_legacy_queue = mes_v10_1_unmap_legacy_queue,
0374     .suspend_gang = mes_v10_1_suspend_gang,
0375     .resume_gang = mes_v10_1_resume_gang,
0376 };
0377 
0378 static int mes_v10_1_init_microcode(struct amdgpu_device *adev,
0379                     enum admgpu_mes_pipe pipe)
0380 {
0381     const char *chip_name;
0382     char fw_name[30];
0383     int err;
0384     const struct mes_firmware_header_v1_0 *mes_hdr;
0385     struct amdgpu_firmware_info *info;
0386 
0387     switch (adev->ip_versions[GC_HWIP][0]) {
0388     case IP_VERSION(10, 1, 10):
0389         chip_name = "navi10";
0390         break;
0391     case IP_VERSION(10, 3, 0):
0392         chip_name = "sienna_cichlid";
0393         break;
0394     default:
0395         BUG();
0396     }
0397 
0398     if (pipe == AMDGPU_MES_SCHED_PIPE)
0399         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes.bin",
0400              chip_name);
0401     else
0402         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mes1.bin",
0403              chip_name);
0404 
0405     err = request_firmware(&adev->mes.fw[pipe], fw_name, adev->dev);
0406     if (err)
0407         return err;
0408 
0409     err = amdgpu_ucode_validate(adev->mes.fw[pipe]);
0410     if (err) {
0411         release_firmware(adev->mes.fw[pipe]);
0412         adev->mes.fw[pipe] = NULL;
0413         return err;
0414     }
0415 
0416     mes_hdr = (const struct mes_firmware_header_v1_0 *)
0417         adev->mes.fw[pipe]->data;
0418     adev->mes.ucode_fw_version[pipe] =
0419         le32_to_cpu(mes_hdr->mes_ucode_version);
0420     adev->mes.ucode_fw_version[pipe] =
0421         le32_to_cpu(mes_hdr->mes_ucode_data_version);
0422     adev->mes.uc_start_addr[pipe] =
0423         le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) |
0424         ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32);
0425     adev->mes.data_start_addr[pipe] =
0426         le32_to_cpu(mes_hdr->mes_data_start_addr_lo) |
0427         ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32);
0428 
0429     if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
0430         int ucode, ucode_data;
0431 
0432         if (pipe == AMDGPU_MES_SCHED_PIPE) {
0433             ucode = AMDGPU_UCODE_ID_CP_MES;
0434             ucode_data = AMDGPU_UCODE_ID_CP_MES_DATA;
0435         } else {
0436             ucode = AMDGPU_UCODE_ID_CP_MES1;
0437             ucode_data = AMDGPU_UCODE_ID_CP_MES1_DATA;
0438         }
0439 
0440         info = &adev->firmware.ucode[ucode];
0441         info->ucode_id = ucode;
0442         info->fw = adev->mes.fw[pipe];
0443         adev->firmware.fw_size +=
0444             ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes),
0445                   PAGE_SIZE);
0446 
0447         info = &adev->firmware.ucode[ucode_data];
0448         info->ucode_id = ucode_data;
0449         info->fw = adev->mes.fw[pipe];
0450         adev->firmware.fw_size +=
0451             ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes),
0452                   PAGE_SIZE);
0453     }
0454 
0455     return 0;
0456 }
0457 
0458 static void mes_v10_1_free_microcode(struct amdgpu_device *adev,
0459                      enum admgpu_mes_pipe pipe)
0460 {
0461     release_firmware(adev->mes.fw[pipe]);
0462     adev->mes.fw[pipe] = NULL;
0463 }
0464 
0465 static int mes_v10_1_allocate_ucode_buffer(struct amdgpu_device *adev,
0466                        enum admgpu_mes_pipe pipe)
0467 {
0468     int r;
0469     const struct mes_firmware_header_v1_0 *mes_hdr;
0470     const __le32 *fw_data;
0471     unsigned fw_size;
0472 
0473     mes_hdr = (const struct mes_firmware_header_v1_0 *)
0474         adev->mes.fw[pipe]->data;
0475 
0476     fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
0477            le32_to_cpu(mes_hdr->mes_ucode_offset_bytes));
0478     fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes);
0479 
0480     r = amdgpu_bo_create_reserved(adev, fw_size,
0481                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
0482                       &adev->mes.ucode_fw_obj[pipe],
0483                       &adev->mes.ucode_fw_gpu_addr[pipe],
0484                       (void **)&adev->mes.ucode_fw_ptr[pipe]);
0485     if (r) {
0486         dev_err(adev->dev, "(%d) failed to create mes fw bo\n", r);
0487         return r;
0488     }
0489 
0490     memcpy(adev->mes.ucode_fw_ptr[pipe], fw_data, fw_size);
0491 
0492     amdgpu_bo_kunmap(adev->mes.ucode_fw_obj[pipe]);
0493     amdgpu_bo_unreserve(adev->mes.ucode_fw_obj[pipe]);
0494 
0495     return 0;
0496 }
0497 
0498 static int mes_v10_1_allocate_ucode_data_buffer(struct amdgpu_device *adev,
0499                         enum admgpu_mes_pipe pipe)
0500 {
0501     int r;
0502     const struct mes_firmware_header_v1_0 *mes_hdr;
0503     const __le32 *fw_data;
0504     unsigned fw_size;
0505 
0506     mes_hdr = (const struct mes_firmware_header_v1_0 *)
0507         adev->mes.fw[pipe]->data;
0508 
0509     fw_data = (const __le32 *)(adev->mes.fw[pipe]->data +
0510            le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes));
0511     fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes);
0512 
0513     r = amdgpu_bo_create_reserved(adev, fw_size,
0514                       64 * 1024, AMDGPU_GEM_DOMAIN_GTT,
0515                       &adev->mes.data_fw_obj[pipe],
0516                       &adev->mes.data_fw_gpu_addr[pipe],
0517                       (void **)&adev->mes.data_fw_ptr[pipe]);
0518     if (r) {
0519         dev_err(adev->dev, "(%d) failed to create mes data fw bo\n", r);
0520         return r;
0521     }
0522 
0523     memcpy(adev->mes.data_fw_ptr[pipe], fw_data, fw_size);
0524 
0525     amdgpu_bo_kunmap(adev->mes.data_fw_obj[pipe]);
0526     amdgpu_bo_unreserve(adev->mes.data_fw_obj[pipe]);
0527 
0528     return 0;
0529 }
0530 
0531 static void mes_v10_1_free_ucode_buffers(struct amdgpu_device *adev,
0532                      enum admgpu_mes_pipe pipe)
0533 {
0534     amdgpu_bo_free_kernel(&adev->mes.data_fw_obj[pipe],
0535                   &adev->mes.data_fw_gpu_addr[pipe],
0536                   (void **)&adev->mes.data_fw_ptr[pipe]);
0537 
0538     amdgpu_bo_free_kernel(&adev->mes.ucode_fw_obj[pipe],
0539                   &adev->mes.ucode_fw_gpu_addr[pipe],
0540                   (void **)&adev->mes.ucode_fw_ptr[pipe]);
0541 }
0542 
0543 static void mes_v10_1_enable(struct amdgpu_device *adev, bool enable)
0544 {
0545     uint32_t pipe, data = 0;
0546 
0547     if (enable) {
0548         data = RREG32_SOC15(GC, 0, mmCP_MES_CNTL);
0549         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
0550         data = REG_SET_FIELD(data, CP_MES_CNTL,
0551                  MES_PIPE1_RESET, adev->enable_mes_kiq ? 1 : 0);
0552         WREG32_SOC15(GC, 0, mmCP_MES_CNTL, data);
0553 
0554         mutex_lock(&adev->srbm_mutex);
0555         for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
0556             if (!adev->enable_mes_kiq &&
0557                 pipe == AMDGPU_MES_KIQ_PIPE)
0558                 continue;
0559 
0560             nv_grbm_select(adev, 3, pipe, 0, 0);
0561             WREG32_SOC15(GC, 0, mmCP_MES_PRGRM_CNTR_START,
0562                  (uint32_t)(adev->mes.uc_start_addr[pipe]) >> 2);
0563         }
0564         nv_grbm_select(adev, 0, 0, 0, 0);
0565         mutex_unlock(&adev->srbm_mutex);
0566 
0567         /* clear BYPASS_UNCACHED to avoid hangs after interrupt. */
0568         data = RREG32_SOC15(GC, 0, mmCP_MES_DC_OP_CNTL);
0569         data = REG_SET_FIELD(data, CP_MES_DC_OP_CNTL,
0570                      BYPASS_UNCACHED, 0);
0571         WREG32_SOC15(GC, 0, mmCP_MES_DC_OP_CNTL, data);
0572 
0573         /* unhalt MES and activate pipe0 */
0574         data = REG_SET_FIELD(0, CP_MES_CNTL, MES_PIPE0_ACTIVE, 1);
0575         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE,
0576                      adev->enable_mes_kiq ? 1 : 0);
0577         WREG32_SOC15(GC, 0, mmCP_MES_CNTL, data);
0578         udelay(100);
0579     } else {
0580         data = RREG32_SOC15(GC, 0, mmCP_MES_CNTL);
0581         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_ACTIVE, 0);
0582         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_ACTIVE, 0);
0583         data = REG_SET_FIELD(data, CP_MES_CNTL,
0584                      MES_INVALIDATE_ICACHE, 1);
0585         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE0_RESET, 1);
0586         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_PIPE1_RESET,
0587                      adev->enable_mes_kiq ? 1 : 0);
0588         data = REG_SET_FIELD(data, CP_MES_CNTL, MES_HALT, 1);
0589         WREG32_SOC15(GC, 0, mmCP_MES_CNTL, data);
0590     }
0591 }
0592 
0593 /* This function is for backdoor MES firmware */
0594 static int mes_v10_1_load_microcode(struct amdgpu_device *adev,
0595                     enum admgpu_mes_pipe pipe)
0596 {
0597     int r;
0598     uint32_t data;
0599 
0600     mes_v10_1_enable(adev, false);
0601 
0602     if (!adev->mes.fw[pipe])
0603         return -EINVAL;
0604 
0605     r = mes_v10_1_allocate_ucode_buffer(adev, pipe);
0606     if (r)
0607         return r;
0608 
0609     r = mes_v10_1_allocate_ucode_data_buffer(adev, pipe);
0610     if (r) {
0611         mes_v10_1_free_ucode_buffers(adev, pipe);
0612         return r;
0613     }
0614 
0615     WREG32_SOC15(GC, 0, mmCP_MES_IC_BASE_CNTL, 0);
0616 
0617     mutex_lock(&adev->srbm_mutex);
0618     /* me=3, pipe=0, queue=0 */
0619     nv_grbm_select(adev, 3, pipe, 0, 0);
0620 
0621     /* set ucode start address */
0622     WREG32_SOC15(GC, 0, mmCP_MES_PRGRM_CNTR_START,
0623              (uint32_t)(adev->mes.uc_start_addr[pipe]) >> 2);
0624 
0625     /* set ucode fimrware address */
0626     WREG32_SOC15(GC, 0, mmCP_MES_IC_BASE_LO,
0627              lower_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
0628     WREG32_SOC15(GC, 0, mmCP_MES_IC_BASE_HI,
0629              upper_32_bits(adev->mes.ucode_fw_gpu_addr[pipe]));
0630 
0631     /* set ucode instruction cache boundary to 2M-1 */
0632     WREG32_SOC15(GC, 0, mmCP_MES_MIBOUND_LO, 0x1FFFFF);
0633 
0634     /* set ucode data firmware address */
0635     WREG32_SOC15(GC, 0, mmCP_MES_MDBASE_LO,
0636              lower_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
0637     WREG32_SOC15(GC, 0, mmCP_MES_MDBASE_HI,
0638              upper_32_bits(adev->mes.data_fw_gpu_addr[pipe]));
0639 
0640     /* Set 0x3FFFF (256K-1) to CP_MES_MDBOUND_LO */
0641     WREG32_SOC15(GC, 0, mmCP_MES_MDBOUND_LO, 0x3FFFF);
0642 
0643     /* invalidate ICACHE */
0644     switch (adev->ip_versions[GC_HWIP][0]) {
0645     case IP_VERSION(10, 3, 0):
0646         data = RREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL_Sienna_Cichlid);
0647         break;
0648     default:
0649         data = RREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL);
0650         break;
0651     }
0652     data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 0);
0653     data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, INVALIDATE_CACHE, 1);
0654     switch (adev->ip_versions[GC_HWIP][0]) {
0655     case IP_VERSION(10, 3, 0):
0656         WREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL_Sienna_Cichlid, data);
0657         break;
0658     default:
0659         WREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL, data);
0660         break;
0661     }
0662 
0663     /* prime the ICACHE. */
0664     switch (adev->ip_versions[GC_HWIP][0]) {
0665     case IP_VERSION(10, 3, 0):
0666         data = RREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL_Sienna_Cichlid);
0667         break;
0668     default:
0669         data = RREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL);
0670         break;
0671     }
0672     data = REG_SET_FIELD(data, CP_MES_IC_OP_CNTL, PRIME_ICACHE, 1);
0673     switch (adev->ip_versions[GC_HWIP][0]) {
0674     case IP_VERSION(10, 3, 0):
0675         WREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL_Sienna_Cichlid, data);
0676         break;
0677     default:
0678         WREG32_SOC15(GC, 0, mmCP_MES_IC_OP_CNTL, data);
0679         break;
0680     }
0681 
0682     nv_grbm_select(adev, 0, 0, 0, 0);
0683     mutex_unlock(&adev->srbm_mutex);
0684 
0685     return 0;
0686 }
0687 
0688 static int mes_v10_1_allocate_eop_buf(struct amdgpu_device *adev,
0689                       enum admgpu_mes_pipe pipe)
0690 {
0691     int r;
0692     u32 *eop;
0693 
0694     r = amdgpu_bo_create_reserved(adev, MES_EOP_SIZE, PAGE_SIZE,
0695                   AMDGPU_GEM_DOMAIN_GTT,
0696                   &adev->mes.eop_gpu_obj[pipe],
0697                   &adev->mes.eop_gpu_addr[pipe],
0698                   (void **)&eop);
0699     if (r) {
0700         dev_warn(adev->dev, "(%d) create EOP bo failed\n", r);
0701         return r;
0702     }
0703 
0704     memset(eop, 0, adev->mes.eop_gpu_obj[pipe]->tbo.base.size);
0705 
0706     amdgpu_bo_kunmap(adev->mes.eop_gpu_obj[pipe]);
0707     amdgpu_bo_unreserve(adev->mes.eop_gpu_obj[pipe]);
0708 
0709     return 0;
0710 }
0711 
0712 static int mes_v10_1_mqd_init(struct amdgpu_ring *ring)
0713 {
0714     struct v10_compute_mqd *mqd = ring->mqd_ptr;
0715     uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
0716     uint32_t tmp;
0717 
0718     mqd->header = 0xC0310800;
0719     mqd->compute_pipelinestat_enable = 0x00000001;
0720     mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
0721     mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
0722     mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
0723     mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
0724     mqd->compute_misc_reserved = 0x00000003;
0725 
0726     eop_base_addr = ring->eop_gpu_addr >> 8;
0727 
0728     /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
0729     tmp = mmCP_HQD_EOP_CONTROL_DEFAULT;
0730     tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
0731             (order_base_2(MES_EOP_SIZE / 4) - 1));
0732 
0733     mqd->cp_hqd_eop_base_addr_lo = lower_32_bits(eop_base_addr);
0734     mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
0735     mqd->cp_hqd_eop_control = tmp;
0736 
0737     /* disable the queue if it's active */
0738     ring->wptr = 0;
0739     mqd->cp_hqd_pq_rptr = 0;
0740     mqd->cp_hqd_pq_wptr_lo = 0;
0741     mqd->cp_hqd_pq_wptr_hi = 0;
0742 
0743     /* set the pointer to the MQD */
0744     mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
0745     mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
0746 
0747     /* set MQD vmid to 0 */
0748     tmp = mmCP_MQD_CONTROL_DEFAULT;
0749     tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
0750     mqd->cp_mqd_control = tmp;
0751 
0752     /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
0753     hqd_gpu_addr = ring->gpu_addr >> 8;
0754     mqd->cp_hqd_pq_base_lo = lower_32_bits(hqd_gpu_addr);
0755     mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
0756 
0757     /* set the wb address whether it's enabled or not */
0758     wb_gpu_addr = ring->rptr_gpu_addr;
0759     mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
0760     mqd->cp_hqd_pq_rptr_report_addr_hi =
0761         upper_32_bits(wb_gpu_addr) & 0xffff;
0762 
0763     /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
0764     wb_gpu_addr = ring->wptr_gpu_addr;
0765     mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffff8;
0766     mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
0767 
0768     /* set up the HQD, this is similar to CP_RB0_CNTL */
0769     tmp = mmCP_HQD_PQ_CONTROL_DEFAULT;
0770     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
0771                 (order_base_2(ring->ring_size / 4) - 1));
0772     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
0773                 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
0774 #ifdef __BIG_ENDIAN
0775     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
0776 #endif
0777     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1);
0778     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
0779     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
0780     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
0781     tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, NO_UPDATE_RPTR, 1);
0782     mqd->cp_hqd_pq_control = tmp;
0783 
0784     /* enable doorbell? */
0785     tmp = 0;
0786     if (ring->use_doorbell) {
0787         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0788                     DOORBELL_OFFSET, ring->doorbell_index);
0789         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0790                     DOORBELL_EN, 1);
0791         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0792                     DOORBELL_SOURCE, 0);
0793         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0794                     DOORBELL_HIT, 0);
0795     }
0796     else
0797         tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
0798                     DOORBELL_EN, 0);
0799     mqd->cp_hqd_pq_doorbell_control = tmp;
0800 
0801     mqd->cp_hqd_vmid = 0;
0802     /* activate the queue */
0803     mqd->cp_hqd_active = 1;
0804     mqd->cp_hqd_persistent_state = mmCP_HQD_PERSISTENT_STATE_DEFAULT;
0805     mqd->cp_hqd_ib_control = mmCP_HQD_IB_CONTROL_DEFAULT;
0806     mqd->cp_hqd_iq_timer = mmCP_HQD_IQ_TIMER_DEFAULT;
0807     mqd->cp_hqd_quantum = mmCP_HQD_QUANTUM_DEFAULT;
0808 
0809     tmp = mmCP_HQD_GFX_CONTROL_DEFAULT;
0810     tmp = REG_SET_FIELD(tmp, CP_HQD_GFX_CONTROL, DB_UPDATED_MSG_EN, 1);
0811     /* offset: 184 - this is used for CP_HQD_GFX_CONTROL */
0812     mqd->cp_hqd_suspend_cntl_stack_offset = tmp;
0813 
0814     return 0;
0815 }
0816 
0817 #if 0
0818 static void mes_v10_1_queue_init_register(struct amdgpu_ring *ring)
0819 {
0820     struct v10_compute_mqd *mqd = ring->mqd_ptr;
0821     struct amdgpu_device *adev = ring->adev;
0822     uint32_t data = 0;
0823 
0824     mutex_lock(&adev->srbm_mutex);
0825     nv_grbm_select(adev, 3, ring->pipe, 0, 0);
0826 
0827     /* set CP_HQD_VMID.VMID = 0. */
0828     data = RREG32_SOC15(GC, 0, mmCP_HQD_VMID);
0829     data = REG_SET_FIELD(data, CP_HQD_VMID, VMID, 0);
0830     WREG32_SOC15(GC, 0, mmCP_HQD_VMID, data);
0831 
0832     /* set CP_HQD_PQ_DOORBELL_CONTROL.DOORBELL_EN=0 */
0833     data = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
0834     data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL,
0835                  DOORBELL_EN, 0);
0836     WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, data);
0837 
0838     /* set CP_MQD_BASE_ADDR/HI with the MQD base address */
0839     WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
0840     WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
0841 
0842     /* set CP_MQD_CONTROL.VMID=0 */
0843     data = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
0844     data = REG_SET_FIELD(data, CP_MQD_CONTROL, VMID, 0);
0845     WREG32_SOC15(GC, 0, mmCP_MQD_CONTROL, 0);
0846 
0847     /* set CP_HQD_PQ_BASE/HI with the ring buffer base address */
0848     WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
0849     WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
0850 
0851     /* set CP_HQD_PQ_RPTR_REPORT_ADDR/HI */
0852     WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
0853              mqd->cp_hqd_pq_rptr_report_addr_lo);
0854     WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
0855              mqd->cp_hqd_pq_rptr_report_addr_hi);
0856 
0857     /* set CP_HQD_PQ_CONTROL */
0858     WREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
0859 
0860     /* set CP_HQD_PQ_WPTR_POLL_ADDR/HI */
0861     WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
0862              mqd->cp_hqd_pq_wptr_poll_addr_lo);
0863     WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
0864              mqd->cp_hqd_pq_wptr_poll_addr_hi);
0865 
0866     /* set CP_HQD_PQ_DOORBELL_CONTROL */
0867     WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
0868              mqd->cp_hqd_pq_doorbell_control);
0869 
0870     /* set CP_HQD_PERSISTENT_STATE.PRELOAD_SIZE=0x53 */
0871     WREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
0872 
0873     /* set CP_HQD_ACTIVE.ACTIVE=1 */
0874     WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, mqd->cp_hqd_active);
0875 
0876     nv_grbm_select(adev, 0, 0, 0, 0);
0877     mutex_unlock(&adev->srbm_mutex);
0878 }
0879 #endif
0880 
0881 static int mes_v10_1_kiq_enable_queue(struct amdgpu_device *adev)
0882 {
0883     struct amdgpu_kiq *kiq = &adev->gfx.kiq;
0884     struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
0885     int r;
0886 
0887     if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
0888         return -EINVAL;
0889 
0890     r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size);
0891     if (r) {
0892         DRM_ERROR("Failed to lock KIQ (%d).\n", r);
0893         return r;
0894     }
0895 
0896     kiq->pmf->kiq_map_queues(kiq_ring, &adev->mes.ring);
0897 
0898     r = amdgpu_ring_test_ring(kiq_ring);
0899     if (r) {
0900         DRM_ERROR("kfq enable failed\n");
0901         kiq_ring->sched.ready = false;
0902     }
0903 
0904     return r;
0905 }
0906 
0907 static int mes_v10_1_queue_init(struct amdgpu_device *adev)
0908 {
0909     int r;
0910 
0911     r = mes_v10_1_mqd_init(&adev->mes.ring);
0912     if (r)
0913         return r;
0914 
0915     r = mes_v10_1_kiq_enable_queue(adev);
0916     if (r)
0917         return r;
0918 
0919     return 0;
0920 }
0921 
0922 static int mes_v10_1_ring_init(struct amdgpu_device *adev)
0923 {
0924     struct amdgpu_ring *ring;
0925 
0926     ring = &adev->mes.ring;
0927 
0928     ring->funcs = &mes_v10_1_ring_funcs;
0929 
0930     ring->me = 3;
0931     ring->pipe = 0;
0932     ring->queue = 0;
0933 
0934     ring->ring_obj = NULL;
0935     ring->use_doorbell = true;
0936     ring->doorbell_index = adev->doorbell_index.mes_ring0 << 1;
0937     ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_SCHED_PIPE];
0938     ring->no_scheduler = true;
0939     sprintf(ring->name, "mes_%d.%d.%d", ring->me, ring->pipe, ring->queue);
0940 
0941     return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
0942                 AMDGPU_RING_PRIO_DEFAULT, NULL);
0943 }
0944 
0945 static int mes_v10_1_kiq_ring_init(struct amdgpu_device *adev)
0946 {
0947     struct amdgpu_ring *ring;
0948 
0949     spin_lock_init(&adev->gfx.kiq.ring_lock);
0950 
0951     ring = &adev->gfx.kiq.ring;
0952 
0953     ring->me = 3;
0954     ring->pipe = 1;
0955     ring->queue = 0;
0956 
0957     ring->adev = NULL;
0958     ring->ring_obj = NULL;
0959     ring->use_doorbell = true;
0960     ring->doorbell_index = adev->doorbell_index.mes_ring1 << 1;
0961     ring->eop_gpu_addr = adev->mes.eop_gpu_addr[AMDGPU_MES_KIQ_PIPE];
0962     ring->no_scheduler = true;
0963     sprintf(ring->name, "mes_kiq_%d.%d.%d",
0964         ring->me, ring->pipe, ring->queue);
0965 
0966     return amdgpu_ring_init(adev, ring, 1024, NULL, 0,
0967                 AMDGPU_RING_PRIO_DEFAULT, NULL);
0968 }
0969 
0970 static int mes_v10_1_mqd_sw_init(struct amdgpu_device *adev,
0971                  enum admgpu_mes_pipe pipe)
0972 {
0973     int r, mqd_size = sizeof(struct v10_compute_mqd);
0974     struct amdgpu_ring *ring;
0975 
0976     if (pipe == AMDGPU_MES_KIQ_PIPE)
0977         ring = &adev->gfx.kiq.ring;
0978     else if (pipe == AMDGPU_MES_SCHED_PIPE)
0979         ring = &adev->mes.ring;
0980     else
0981         BUG();
0982 
0983     if (ring->mqd_obj)
0984         return 0;
0985 
0986     r = amdgpu_bo_create_kernel(adev, mqd_size, PAGE_SIZE,
0987                     AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
0988                     &ring->mqd_gpu_addr, &ring->mqd_ptr);
0989     if (r) {
0990         dev_warn(adev->dev, "failed to create ring mqd bo (%d)", r);
0991         return r;
0992     }
0993     memset(ring->mqd_ptr, 0, mqd_size);
0994 
0995     /* prepare MQD backup */
0996     adev->mes.mqd_backup[pipe] = kmalloc(mqd_size, GFP_KERNEL);
0997     if (!adev->mes.mqd_backup[pipe])
0998         dev_warn(adev->dev,
0999              "no memory to create MQD backup for ring %s\n",
1000              ring->name);
1001 
1002     return 0;
1003 }
1004 
1005 static int mes_v10_1_sw_init(void *handle)
1006 {
1007     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1008     int pipe, r;
1009 
1010     adev->mes.adev = adev;
1011     adev->mes.funcs = &mes_v10_1_funcs;
1012     adev->mes.kiq_hw_init = &mes_v10_1_kiq_hw_init;
1013 
1014     r = amdgpu_mes_init(adev);
1015     if (r)
1016         return r;
1017 
1018     for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1019         if (!adev->enable_mes_kiq && pipe == AMDGPU_MES_KIQ_PIPE)
1020             continue;
1021 
1022         r = mes_v10_1_init_microcode(adev, pipe);
1023         if (r)
1024             return r;
1025 
1026         r = mes_v10_1_allocate_eop_buf(adev, pipe);
1027         if (r)
1028             return r;
1029 
1030         r = mes_v10_1_mqd_sw_init(adev, pipe);
1031         if (r)
1032             return r;
1033     }
1034 
1035     if (adev->enable_mes_kiq) {
1036         r = mes_v10_1_kiq_ring_init(adev);
1037         if (r)
1038             return r;
1039     }
1040 
1041     r = mes_v10_1_ring_init(adev);
1042     if (r)
1043         return r;
1044 
1045     return 0;
1046 }
1047 
1048 static int mes_v10_1_sw_fini(void *handle)
1049 {
1050     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1051     int pipe;
1052 
1053     amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs);
1054     amdgpu_device_wb_free(adev, adev->mes.query_status_fence_offs);
1055 
1056     for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
1057         kfree(adev->mes.mqd_backup[pipe]);
1058 
1059         amdgpu_bo_free_kernel(&adev->mes.eop_gpu_obj[pipe],
1060                       &adev->mes.eop_gpu_addr[pipe],
1061                       NULL);
1062 
1063         mes_v10_1_free_microcode(adev, pipe);
1064     }
1065 
1066     amdgpu_bo_free_kernel(&adev->gfx.kiq.ring.mqd_obj,
1067                   &adev->gfx.kiq.ring.mqd_gpu_addr,
1068                   &adev->gfx.kiq.ring.mqd_ptr);
1069 
1070     amdgpu_bo_free_kernel(&adev->mes.ring.mqd_obj,
1071                   &adev->mes.ring.mqd_gpu_addr,
1072                   &adev->mes.ring.mqd_ptr);
1073 
1074     amdgpu_ring_fini(&adev->gfx.kiq.ring);
1075     amdgpu_ring_fini(&adev->mes.ring);
1076 
1077     amdgpu_mes_fini(adev);
1078     return 0;
1079 }
1080 
1081 static void mes_v10_1_kiq_setting(struct amdgpu_ring *ring)
1082 {
1083     uint32_t tmp;
1084     struct amdgpu_device *adev = ring->adev;
1085 
1086     /* tell RLC which is KIQ queue */
1087     switch (adev->ip_versions[GC_HWIP][0]) {
1088     case IP_VERSION(10, 3, 0):
1089     case IP_VERSION(10, 3, 2):
1090     case IP_VERSION(10, 3, 1):
1091     case IP_VERSION(10, 3, 4):
1092         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS_Sienna_Cichlid);
1093         tmp &= 0xffffff00;
1094         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1095         WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS_Sienna_Cichlid, tmp);
1096         tmp |= 0x80;
1097         WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS_Sienna_Cichlid, tmp);
1098         break;
1099     default:
1100         tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
1101         tmp &= 0xffffff00;
1102         tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
1103         WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
1104         tmp |= 0x80;
1105         WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
1106         break;
1107     }
1108 }
1109 
1110 static int mes_v10_1_kiq_hw_init(struct amdgpu_device *adev)
1111 {
1112     int r = 0;
1113 
1114     if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1115         r = mes_v10_1_load_microcode(adev, AMDGPU_MES_KIQ_PIPE);
1116         if (r) {
1117             DRM_ERROR("failed to load MES kiq fw, r=%d\n", r);
1118             return r;
1119         }
1120 
1121         r = mes_v10_1_load_microcode(adev, AMDGPU_MES_SCHED_PIPE);
1122         if (r) {
1123             DRM_ERROR("failed to load MES fw, r=%d\n", r);
1124             return r;
1125         }
1126     }
1127 
1128     mes_v10_1_enable(adev, true);
1129 
1130     mes_v10_1_kiq_setting(&adev->gfx.kiq.ring);
1131 
1132     r = mes_v10_1_queue_init(adev);
1133     if (r)
1134         goto failure;
1135 
1136     return r;
1137 
1138 failure:
1139     mes_v10_1_hw_fini(adev);
1140     return r;
1141 }
1142 
1143 static int mes_v10_1_hw_init(void *handle)
1144 {
1145     int r;
1146     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1147 
1148     if (!adev->enable_mes_kiq) {
1149         if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1150             r = mes_v10_1_load_microcode(adev,
1151                          AMDGPU_MES_SCHED_PIPE);
1152             if (r) {
1153                 DRM_ERROR("failed to MES fw, r=%d\n", r);
1154                 return r;
1155             }
1156         }
1157 
1158         mes_v10_1_enable(adev, true);
1159     }
1160 
1161     r = mes_v10_1_queue_init(adev);
1162     if (r)
1163         goto failure;
1164 
1165     r = mes_v10_1_set_hw_resources(&adev->mes);
1166     if (r)
1167         goto failure;
1168 
1169     mes_v10_1_init_aggregated_doorbell(&adev->mes);
1170 
1171     r = mes_v10_1_query_sched_status(&adev->mes);
1172     if (r) {
1173         DRM_ERROR("MES is busy\n");
1174         goto failure;
1175     }
1176 
1177     /*
1178      * Disable KIQ ring usage from the driver once MES is enabled.
1179      * MES uses KIQ ring exclusively so driver cannot access KIQ ring
1180      * with MES enabled.
1181      */
1182     adev->gfx.kiq.ring.sched.ready = false;
1183     adev->mes.ring.sched.ready = true;
1184 
1185     return 0;
1186 
1187 failure:
1188     mes_v10_1_hw_fini(adev);
1189     return r;
1190 }
1191 
1192 static int mes_v10_1_hw_fini(void *handle)
1193 {
1194     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1195 
1196     adev->mes.ring.sched.ready = false;
1197 
1198     mes_v10_1_enable(adev, false);
1199 
1200     if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1201         mes_v10_1_free_ucode_buffers(adev, AMDGPU_MES_KIQ_PIPE);
1202         mes_v10_1_free_ucode_buffers(adev, AMDGPU_MES_SCHED_PIPE);
1203     }
1204 
1205     return 0;
1206 }
1207 
1208 static int mes_v10_1_suspend(void *handle)
1209 {
1210     int r;
1211     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1212 
1213     r = amdgpu_mes_suspend(adev);
1214     if (r)
1215         return r;
1216 
1217     return mes_v10_1_hw_fini(adev);
1218 }
1219 
1220 static int mes_v10_1_resume(void *handle)
1221 {
1222     int r;
1223     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1224 
1225     r = mes_v10_1_hw_init(adev);
1226     if (r)
1227         return r;
1228 
1229     return amdgpu_mes_resume(adev);
1230 }
1231 
1232 static int mes_v10_0_late_init(void *handle)
1233 {
1234     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1235 
1236     if (!amdgpu_in_reset(adev))
1237         amdgpu_mes_self_test(adev);
1238 
1239     return 0;
1240 }
1241 
1242 static const struct amd_ip_funcs mes_v10_1_ip_funcs = {
1243     .name = "mes_v10_1",
1244     .late_init = mes_v10_0_late_init,
1245     .sw_init = mes_v10_1_sw_init,
1246     .sw_fini = mes_v10_1_sw_fini,
1247     .hw_init = mes_v10_1_hw_init,
1248     .hw_fini = mes_v10_1_hw_fini,
1249     .suspend = mes_v10_1_suspend,
1250     .resume = mes_v10_1_resume,
1251 };
1252 
1253 const struct amdgpu_ip_block_version mes_v10_1_ip_block = {
1254     .type = AMD_IP_BLOCK_TYPE_MES,
1255     .major = 10,
1256     .minor = 1,
1257     .rev = 0,
1258     .funcs = &mes_v10_1_ip_funcs,
1259 };
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!