Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright 2019 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023 
0024 #include <linux/firmware.h>
0025 #include "amdgpu.h"
0026 #include "amdgpu_vcn.h"
0027 #include "amdgpu_pm.h"
0028 #include "amdgpu_cs.h"
0029 #include "soc15.h"
0030 #include "soc15d.h"
0031 #include "vcn_v2_0.h"
0032 #include "mmsch_v3_0.h"
0033 #include "vcn_sw_ring.h"
0034 
0035 #include "vcn/vcn_3_0_0_offset.h"
0036 #include "vcn/vcn_3_0_0_sh_mask.h"
0037 #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
0038 
0039 #include <drm/drm_drv.h>
0040 
0041 #define VCN_VID_SOC_ADDRESS_2_0                 0x1fa00
0042 #define VCN1_VID_SOC_ADDRESS_3_0                0x48200
0043 
0044 #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET            0x27
0045 #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET            0x0f
0046 #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET          0x10
0047 #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET          0x11
0048 #define mmUVD_NO_OP_INTERNAL_OFFSET             0x29
0049 #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET           0x66
0050 #define mmUVD_SCRATCH9_INTERNAL_OFFSET              0xc01d
0051 
0052 #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET           0x431
0053 #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET      0x3b4
0054 #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET     0x3b5
0055 #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET           0x25c
0056 
0057 #define VCN_INSTANCES_SIENNA_CICHLID                2
0058 #define DEC_SW_RING_ENABLED                 FALSE
0059 
0060 #define RDECODE_MSG_CREATE                  0x00000000
0061 #define RDECODE_MESSAGE_CREATE                  0x00000001
0062 
0063 static int amdgpu_ih_clientid_vcns[] = {
0064     SOC15_IH_CLIENTID_VCN,
0065     SOC15_IH_CLIENTID_VCN1
0066 };
0067 
0068 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev);
0069 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev);
0070 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev);
0071 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev);
0072 static int vcn_v3_0_set_powergating_state(void *handle,
0073             enum amd_powergating_state state);
0074 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
0075             int inst_idx, struct dpg_pause_state *new_state);
0076 
0077 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring);
0078 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring);
0079 
0080 /**
0081  * vcn_v3_0_early_init - set function pointers
0082  *
0083  * @handle: amdgpu_device pointer
0084  *
0085  * Set ring and irq function pointers
0086  */
0087 static int vcn_v3_0_early_init(void *handle)
0088 {
0089     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0090 
0091     if (amdgpu_sriov_vf(adev)) {
0092         adev->vcn.num_vcn_inst = VCN_INSTANCES_SIENNA_CICHLID;
0093         adev->vcn.harvest_config = 0;
0094         adev->vcn.num_enc_rings = 1;
0095 
0096     } else {
0097         if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
0098                          AMDGPU_VCN_HARVEST_VCN1))
0099             /* both instances are harvested, disable the block */
0100             return -ENOENT;
0101 
0102         if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 0, 33))
0103             adev->vcn.num_enc_rings = 0;
0104         else
0105             adev->vcn.num_enc_rings = 2;
0106     }
0107 
0108     vcn_v3_0_set_dec_ring_funcs(adev);
0109     vcn_v3_0_set_enc_ring_funcs(adev);
0110     vcn_v3_0_set_irq_funcs(adev);
0111 
0112     return 0;
0113 }
0114 
0115 /**
0116  * vcn_v3_0_sw_init - sw init for VCN block
0117  *
0118  * @handle: amdgpu_device pointer
0119  *
0120  * Load firmware and sw initialization
0121  */
0122 static int vcn_v3_0_sw_init(void *handle)
0123 {
0124     struct amdgpu_ring *ring;
0125     int i, j, r;
0126     int vcn_doorbell_index = 0;
0127     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0128 
0129     r = amdgpu_vcn_sw_init(adev);
0130     if (r)
0131         return r;
0132 
0133     amdgpu_vcn_setup_ucode(adev);
0134 
0135     r = amdgpu_vcn_resume(adev);
0136     if (r)
0137         return r;
0138 
0139     /*
0140      * Note: doorbell assignment is fixed for SRIOV multiple VCN engines
0141      * Formula:
0142      *   vcn_db_base  = adev->doorbell_index.vcn.vcn_ring0_1 << 1;
0143      *   dec_ring_i   = vcn_db_base + i * (adev->vcn.num_enc_rings + 1)
0144      *   enc_ring_i,j = vcn_db_base + i * (adev->vcn.num_enc_rings + 1) + 1 + j
0145      */
0146     if (amdgpu_sriov_vf(adev)) {
0147         vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1;
0148         /* get DWORD offset */
0149         vcn_doorbell_index = vcn_doorbell_index << 1;
0150     }
0151 
0152     for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
0153         volatile struct amdgpu_fw_shared *fw_shared;
0154 
0155         if (adev->vcn.harvest_config & (1 << i))
0156             continue;
0157 
0158         adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
0159         adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
0160         adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
0161         adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
0162         adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
0163         adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
0164 
0165         adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
0166         adev->vcn.inst[i].external.scratch9 = SOC15_REG_OFFSET(VCN, i, mmUVD_SCRATCH9);
0167         adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
0168         adev->vcn.inst[i].external.data0 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA0);
0169         adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
0170         adev->vcn.inst[i].external.data1 = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_DATA1);
0171         adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
0172         adev->vcn.inst[i].external.cmd = SOC15_REG_OFFSET(VCN, i, mmUVD_GPCOM_VCPU_CMD);
0173         adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
0174         adev->vcn.inst[i].external.nop = SOC15_REG_OFFSET(VCN, i, mmUVD_NO_OP);
0175 
0176         /* VCN DEC TRAP */
0177         r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
0178                 VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[i].irq);
0179         if (r)
0180             return r;
0181 
0182         atomic_set(&adev->vcn.inst[i].sched_score, 0);
0183 
0184         ring = &adev->vcn.inst[i].ring_dec;
0185         ring->use_doorbell = true;
0186         if (amdgpu_sriov_vf(adev)) {
0187             ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1);
0188         } else {
0189             ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i;
0190         }
0191         sprintf(ring->name, "vcn_dec_%d", i);
0192         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
0193                      AMDGPU_RING_PRIO_DEFAULT,
0194                      &adev->vcn.inst[i].sched_score);
0195         if (r)
0196             return r;
0197 
0198         for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
0199             enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(j);
0200 
0201             /* VCN ENC TRAP */
0202             r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
0203                 j + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
0204             if (r)
0205                 return r;
0206 
0207             ring = &adev->vcn.inst[i].ring_enc[j];
0208             ring->use_doorbell = true;
0209             if (amdgpu_sriov_vf(adev)) {
0210                 ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1 + j;
0211             } else {
0212                 ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + j + 8 * i;
0213             }
0214             sprintf(ring->name, "vcn_enc_%d.%d", i, j);
0215             r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
0216                          hw_prio, &adev->vcn.inst[i].sched_score);
0217             if (r)
0218                 return r;
0219         }
0220 
0221         fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
0222         fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SW_RING_FLAG) |
0223                          cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG) |
0224                          cpu_to_le32(AMDGPU_VCN_FW_SHARED_FLAG_0_RB);
0225         fw_shared->sw_ring.is_enabled = cpu_to_le32(DEC_SW_RING_ENABLED);
0226         fw_shared->present_flag_0 |= AMDGPU_VCN_SMU_VERSION_INFO_FLAG;
0227         if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 1, 2))
0228             fw_shared->smu_interface_info.smu_interface_type = 2;
0229         else if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(3, 1, 1))
0230             fw_shared->smu_interface_info.smu_interface_type = 1;
0231 
0232         if (amdgpu_vcnfw_log)
0233             amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
0234     }
0235 
0236     if (amdgpu_sriov_vf(adev)) {
0237         r = amdgpu_virt_alloc_mm_table(adev);
0238         if (r)
0239             return r;
0240     }
0241     if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
0242         adev->vcn.pause_dpg_mode = vcn_v3_0_pause_dpg_mode;
0243 
0244     return 0;
0245 }
0246 
0247 /**
0248  * vcn_v3_0_sw_fini - sw fini for VCN block
0249  *
0250  * @handle: amdgpu_device pointer
0251  *
0252  * VCN suspend and free up sw allocation
0253  */
0254 static int vcn_v3_0_sw_fini(void *handle)
0255 {
0256     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0257     int i, r, idx;
0258 
0259     if (drm_dev_enter(adev_to_drm(adev), &idx)) {
0260         for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
0261             volatile struct amdgpu_fw_shared *fw_shared;
0262 
0263             if (adev->vcn.harvest_config & (1 << i))
0264                 continue;
0265             fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
0266             fw_shared->present_flag_0 = 0;
0267             fw_shared->sw_ring.is_enabled = false;
0268         }
0269 
0270         drm_dev_exit(idx);
0271     }
0272 
0273     if (amdgpu_sriov_vf(adev))
0274         amdgpu_virt_free_mm_table(adev);
0275 
0276     r = amdgpu_vcn_suspend(adev);
0277     if (r)
0278         return r;
0279 
0280     r = amdgpu_vcn_sw_fini(adev);
0281 
0282     return r;
0283 }
0284 
0285 /**
0286  * vcn_v3_0_hw_init - start and test VCN block
0287  *
0288  * @handle: amdgpu_device pointer
0289  *
0290  * Initialize the hardware, boot up the VCPU and do some testing
0291  */
0292 static int vcn_v3_0_hw_init(void *handle)
0293 {
0294     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0295     struct amdgpu_ring *ring;
0296     int i, j, r;
0297 
0298     if (amdgpu_sriov_vf(adev)) {
0299         r = vcn_v3_0_start_sriov(adev);
0300         if (r)
0301             goto done;
0302 
0303         /* initialize VCN dec and enc ring buffers */
0304         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
0305             if (adev->vcn.harvest_config & (1 << i))
0306                 continue;
0307 
0308             ring = &adev->vcn.inst[i].ring_dec;
0309             if (amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, i)) {
0310                 ring->sched.ready = false;
0311                 ring->no_scheduler = true;
0312                 dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
0313             } else {
0314                 ring->wptr = 0;
0315                 ring->wptr_old = 0;
0316                 vcn_v3_0_dec_ring_set_wptr(ring);
0317                 ring->sched.ready = true;
0318             }
0319 
0320             for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
0321                 ring = &adev->vcn.inst[i].ring_enc[j];
0322                 if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
0323                     ring->sched.ready = false;
0324                     ring->no_scheduler = true;
0325                     dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
0326                 } else {
0327                     ring->wptr = 0;
0328                     ring->wptr_old = 0;
0329                     vcn_v3_0_enc_ring_set_wptr(ring);
0330                     ring->sched.ready = true;
0331                 }
0332             }
0333         }
0334     } else {
0335         for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
0336             if (adev->vcn.harvest_config & (1 << i))
0337                 continue;
0338 
0339             ring = &adev->vcn.inst[i].ring_dec;
0340 
0341             adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
0342                              ring->doorbell_index, i);
0343 
0344             r = amdgpu_ring_test_helper(ring);
0345             if (r)
0346                 goto done;
0347 
0348             for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
0349                 ring = &adev->vcn.inst[i].ring_enc[j];
0350                 r = amdgpu_ring_test_helper(ring);
0351                 if (r)
0352                     goto done;
0353             }
0354         }
0355     }
0356 
0357 done:
0358     if (!r)
0359         DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
0360             (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
0361 
0362     return r;
0363 }
0364 
0365 /**
0366  * vcn_v3_0_hw_fini - stop the hardware block
0367  *
0368  * @handle: amdgpu_device pointer
0369  *
0370  * Stop the VCN block, mark ring as not ready any more
0371  */
0372 static int vcn_v3_0_hw_fini(void *handle)
0373 {
0374     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0375     int i;
0376 
0377     cancel_delayed_work_sync(&adev->vcn.idle_work);
0378 
0379     for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
0380         if (adev->vcn.harvest_config & (1 << i))
0381             continue;
0382 
0383         if (!amdgpu_sriov_vf(adev)) {
0384             if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
0385                     (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
0386                      RREG32_SOC15(VCN, i, mmUVD_STATUS))) {
0387                 vcn_v3_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
0388             }
0389         }
0390     }
0391 
0392     return 0;
0393 }
0394 
0395 /**
0396  * vcn_v3_0_suspend - suspend VCN block
0397  *
0398  * @handle: amdgpu_device pointer
0399  *
0400  * HW fini and suspend VCN block
0401  */
0402 static int vcn_v3_0_suspend(void *handle)
0403 {
0404     int r;
0405     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0406 
0407     r = vcn_v3_0_hw_fini(adev);
0408     if (r)
0409         return r;
0410 
0411     r = amdgpu_vcn_suspend(adev);
0412 
0413     return r;
0414 }
0415 
0416 /**
0417  * vcn_v3_0_resume - resume VCN block
0418  *
0419  * @handle: amdgpu_device pointer
0420  *
0421  * Resume firmware and hw init VCN block
0422  */
0423 static int vcn_v3_0_resume(void *handle)
0424 {
0425     int r;
0426     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0427 
0428     r = amdgpu_vcn_resume(adev);
0429     if (r)
0430         return r;
0431 
0432     r = vcn_v3_0_hw_init(adev);
0433 
0434     return r;
0435 }
0436 
0437 /**
0438  * vcn_v3_0_mc_resume - memory controller programming
0439  *
0440  * @adev: amdgpu_device pointer
0441  * @inst: instance number
0442  *
0443  * Let the VCN memory controller know it's offsets
0444  */
0445 static void vcn_v3_0_mc_resume(struct amdgpu_device *adev, int inst)
0446 {
0447     uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
0448     uint32_t offset;
0449 
0450     /* cache window 0: fw */
0451     if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
0452         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
0453             (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
0454         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
0455             (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
0456         WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0, 0);
0457         offset = 0;
0458     } else {
0459         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
0460             lower_32_bits(adev->vcn.inst[inst].gpu_addr));
0461         WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
0462             upper_32_bits(adev->vcn.inst[inst].gpu_addr));
0463         offset = size;
0464         WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET0,
0465             AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
0466     }
0467     WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE0, size);
0468 
0469     /* cache window 1: stack */
0470     WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
0471         lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
0472     WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
0473         upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
0474     WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET1, 0);
0475     WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
0476 
0477     /* cache window 2: context */
0478     WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
0479         lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
0480     WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
0481         upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
0482     WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_OFFSET2, 0);
0483     WREG32_SOC15(VCN, inst, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
0484 
0485     /* non-cache window */
0486     WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
0487         lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
0488     WREG32_SOC15(VCN, inst, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
0489         upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
0490     WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
0491     WREG32_SOC15(VCN, inst, mmUVD_VCPU_NONCACHE_SIZE0,
0492         AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
0493 }
0494 
0495 static void vcn_v3_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
0496 {
0497     uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
0498     uint32_t offset;
0499 
0500     /* cache window 0: fw */
0501     if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
0502         if (!indirect) {
0503             WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0504                 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
0505                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
0506             WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0507                 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
0508                 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
0509             WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0510                 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
0511         } else {
0512             WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0513                 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
0514             WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0515                 VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
0516             WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0517                 VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
0518         }
0519         offset = 0;
0520     } else {
0521         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0522             VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
0523             lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
0524         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0525             VCN, inst_idx, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
0526             upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
0527         offset = size;
0528         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0529             VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET0),
0530             AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
0531     }
0532 
0533     if (!indirect)
0534         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0535             VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
0536     else
0537         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0538             VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
0539 
0540     /* cache window 1: stack */
0541     if (!indirect) {
0542         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0543             VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
0544             lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
0545         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0546             VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
0547             upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
0548         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0549             VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
0550     } else {
0551         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0552             VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
0553         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0554             VCN, inst_idx, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
0555         WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0556             VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
0557     }
0558     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0559             VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
0560 
0561     /* cache window 2: context */
0562     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0563             VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
0564             lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
0565     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0566             VCN, inst_idx, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
0567             upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
0568     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0569             VCN, inst_idx, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
0570     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0571             VCN, inst_idx, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
0572 
0573     /* non-cache window */
0574     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0575             VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
0576             lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
0577     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0578             VCN, inst_idx, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
0579             upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
0580     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0581             VCN, inst_idx, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
0582     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0583             VCN, inst_idx, mmUVD_VCPU_NONCACHE_SIZE0),
0584             AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
0585 
0586     /* VCN global tiling registers */
0587     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0588         UVD, inst_idx, mmUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
0589 }
0590 
0591 static void vcn_v3_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
0592 {
0593     uint32_t data = 0;
0594 
0595     if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
0596         data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
0597             | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
0598             | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
0599             | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
0600             | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
0601             | 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
0602             | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
0603             | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
0604             | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
0605             | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
0606             | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
0607             | 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
0608             | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
0609             | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
0610 
0611         WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
0612         SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS,
0613             UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
0614     } else {
0615         data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
0616             | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
0617             | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
0618             | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
0619             | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
0620             | 1 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
0621             | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
0622             | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
0623             | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
0624             | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
0625             | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
0626             | 1 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
0627             | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
0628             | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
0629         WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
0630         SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, 0,  0x3F3FFFFF);
0631     }
0632 
0633     data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
0634     data &= ~0x103;
0635     if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
0636         data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
0637             UVD_POWER_STATUS__UVD_PG_EN_MASK;
0638 
0639     WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
0640 }
0641 
0642 static void vcn_v3_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
0643 {
0644     uint32_t data;
0645 
0646     if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
0647         /* Before power off, this indicator has to be turned on */
0648         data = RREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS);
0649         data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
0650         data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
0651         WREG32_SOC15(VCN, inst, mmUVD_POWER_STATUS, data);
0652 
0653         data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
0654             | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
0655             | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
0656             | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
0657             | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
0658             | 2 << UVD_PGFSM_CONFIG__UVDIRL_PWR_CONFIG__SHIFT
0659             | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
0660             | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
0661             | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
0662             | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
0663             | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
0664             | 2 << UVD_PGFSM_CONFIG__UVDATD_PWR_CONFIG__SHIFT
0665             | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
0666             | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
0667         WREG32_SOC15(VCN, inst, mmUVD_PGFSM_CONFIG, data);
0668 
0669         data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
0670             | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
0671             | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
0672             | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
0673             | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
0674             | 2 << UVD_PGFSM_STATUS__UVDIRL_PWR_STATUS__SHIFT
0675             | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
0676             | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
0677             | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
0678             | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
0679             | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
0680             | 2 << UVD_PGFSM_STATUS__UVDATD_PWR_STATUS__SHIFT
0681             | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
0682             | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
0683         SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
0684     }
0685 }
0686 
0687 /**
0688  * vcn_v3_0_disable_clock_gating - disable VCN clock gating
0689  *
0690  * @adev: amdgpu_device pointer
0691  * @inst: instance number
0692  *
0693  * Disable clock gating for VCN block
0694  */
0695 static void vcn_v3_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
0696 {
0697     uint32_t data;
0698 
0699     /* VCN disable CGC */
0700     data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
0701     if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
0702         data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
0703     else
0704         data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
0705     data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
0706     data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
0707     WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
0708 
0709     data = RREG32_SOC15(VCN, inst, mmUVD_CGC_GATE);
0710     data &= ~(UVD_CGC_GATE__SYS_MASK
0711         | UVD_CGC_GATE__UDEC_MASK
0712         | UVD_CGC_GATE__MPEG2_MASK
0713         | UVD_CGC_GATE__REGS_MASK
0714         | UVD_CGC_GATE__RBC_MASK
0715         | UVD_CGC_GATE__LMI_MC_MASK
0716         | UVD_CGC_GATE__LMI_UMC_MASK
0717         | UVD_CGC_GATE__IDCT_MASK
0718         | UVD_CGC_GATE__MPRD_MASK
0719         | UVD_CGC_GATE__MPC_MASK
0720         | UVD_CGC_GATE__LBSI_MASK
0721         | UVD_CGC_GATE__LRBBM_MASK
0722         | UVD_CGC_GATE__UDEC_RE_MASK
0723         | UVD_CGC_GATE__UDEC_CM_MASK
0724         | UVD_CGC_GATE__UDEC_IT_MASK
0725         | UVD_CGC_GATE__UDEC_DB_MASK
0726         | UVD_CGC_GATE__UDEC_MP_MASK
0727         | UVD_CGC_GATE__WCB_MASK
0728         | UVD_CGC_GATE__VCPU_MASK
0729         | UVD_CGC_GATE__MMSCH_MASK);
0730 
0731     WREG32_SOC15(VCN, inst, mmUVD_CGC_GATE, data);
0732 
0733     SOC15_WAIT_ON_RREG(VCN, inst, mmUVD_CGC_GATE, 0,  0xFFFFFFFF);
0734 
0735     data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
0736     data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
0737         | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
0738         | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
0739         | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
0740         | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
0741         | UVD_CGC_CTRL__SYS_MODE_MASK
0742         | UVD_CGC_CTRL__UDEC_MODE_MASK
0743         | UVD_CGC_CTRL__MPEG2_MODE_MASK
0744         | UVD_CGC_CTRL__REGS_MODE_MASK
0745         | UVD_CGC_CTRL__RBC_MODE_MASK
0746         | UVD_CGC_CTRL__LMI_MC_MODE_MASK
0747         | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
0748         | UVD_CGC_CTRL__IDCT_MODE_MASK
0749         | UVD_CGC_CTRL__MPRD_MODE_MASK
0750         | UVD_CGC_CTRL__MPC_MODE_MASK
0751         | UVD_CGC_CTRL__LBSI_MODE_MASK
0752         | UVD_CGC_CTRL__LRBBM_MODE_MASK
0753         | UVD_CGC_CTRL__WCB_MODE_MASK
0754         | UVD_CGC_CTRL__VCPU_MODE_MASK
0755         | UVD_CGC_CTRL__MMSCH_MODE_MASK);
0756     WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
0757 
0758     data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE);
0759     data |= (UVD_SUVD_CGC_GATE__SRE_MASK
0760         | UVD_SUVD_CGC_GATE__SIT_MASK
0761         | UVD_SUVD_CGC_GATE__SMP_MASK
0762         | UVD_SUVD_CGC_GATE__SCM_MASK
0763         | UVD_SUVD_CGC_GATE__SDB_MASK
0764         | UVD_SUVD_CGC_GATE__SRE_H264_MASK
0765         | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
0766         | UVD_SUVD_CGC_GATE__SIT_H264_MASK
0767         | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
0768         | UVD_SUVD_CGC_GATE__SCM_H264_MASK
0769         | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
0770         | UVD_SUVD_CGC_GATE__SDB_H264_MASK
0771         | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
0772         | UVD_SUVD_CGC_GATE__SCLR_MASK
0773         | UVD_SUVD_CGC_GATE__ENT_MASK
0774         | UVD_SUVD_CGC_GATE__IME_MASK
0775         | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
0776         | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
0777         | UVD_SUVD_CGC_GATE__SITE_MASK
0778         | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
0779         | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
0780         | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
0781         | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
0782         | UVD_SUVD_CGC_GATE__IME_HEVC_MASK
0783         | UVD_SUVD_CGC_GATE__EFC_MASK
0784         | UVD_SUVD_CGC_GATE__SAOE_MASK
0785         | UVD_SUVD_CGC_GATE__SRE_AV1_MASK
0786         | UVD_SUVD_CGC_GATE__FBC_PCLK_MASK
0787         | UVD_SUVD_CGC_GATE__FBC_CCLK_MASK
0788         | UVD_SUVD_CGC_GATE__SCM_AV1_MASK
0789         | UVD_SUVD_CGC_GATE__SMPA_MASK);
0790     WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE, data);
0791 
0792     data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2);
0793     data |= (UVD_SUVD_CGC_GATE2__MPBE0_MASK
0794         | UVD_SUVD_CGC_GATE2__MPBE1_MASK
0795         | UVD_SUVD_CGC_GATE2__SIT_AV1_MASK
0796         | UVD_SUVD_CGC_GATE2__SDB_AV1_MASK
0797         | UVD_SUVD_CGC_GATE2__MPC1_MASK);
0798     WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_GATE2, data);
0799 
0800     data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
0801     data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
0802         | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
0803         | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
0804         | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
0805         | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
0806         | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
0807         | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
0808         | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
0809         | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
0810         | UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
0811         | UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
0812         | UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
0813         | UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
0814         | UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
0815         | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
0816         | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
0817         | UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
0818         | UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
0819         | UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
0820     WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
0821 }
0822 
0823 static void vcn_v3_0_clock_gating_dpg_mode(struct amdgpu_device *adev,
0824         uint8_t sram_sel, int inst_idx, uint8_t indirect)
0825 {
0826     uint32_t reg_data = 0;
0827 
0828     /* enable sw clock gating control */
0829     if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
0830         reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
0831     else
0832         reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
0833     reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
0834     reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
0835     reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
0836          UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
0837          UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
0838          UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
0839          UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
0840          UVD_CGC_CTRL__SYS_MODE_MASK |
0841          UVD_CGC_CTRL__UDEC_MODE_MASK |
0842          UVD_CGC_CTRL__MPEG2_MODE_MASK |
0843          UVD_CGC_CTRL__REGS_MODE_MASK |
0844          UVD_CGC_CTRL__RBC_MODE_MASK |
0845          UVD_CGC_CTRL__LMI_MC_MODE_MASK |
0846          UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
0847          UVD_CGC_CTRL__IDCT_MODE_MASK |
0848          UVD_CGC_CTRL__MPRD_MODE_MASK |
0849          UVD_CGC_CTRL__MPC_MODE_MASK |
0850          UVD_CGC_CTRL__LBSI_MODE_MASK |
0851          UVD_CGC_CTRL__LRBBM_MODE_MASK |
0852          UVD_CGC_CTRL__WCB_MODE_MASK |
0853          UVD_CGC_CTRL__VCPU_MODE_MASK |
0854          UVD_CGC_CTRL__MMSCH_MODE_MASK);
0855     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0856         VCN, inst_idx, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
0857 
0858     /* turn off clock gating */
0859     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0860         VCN, inst_idx, mmUVD_CGC_GATE), 0, sram_sel, indirect);
0861 
0862     /* turn on SUVD clock gating */
0863     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0864         VCN, inst_idx, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
0865 
0866     /* turn on sw mode in UVD_SUVD_CGC_CTRL */
0867     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0868         VCN, inst_idx, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
0869 }
0870 
0871 /**
0872  * vcn_v3_0_enable_clock_gating - enable VCN clock gating
0873  *
0874  * @adev: amdgpu_device pointer
0875  * @inst: instance number
0876  *
0877  * Enable clock gating for VCN block
0878  */
0879 static void vcn_v3_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
0880 {
0881     uint32_t data;
0882 
0883     /* enable VCN CGC */
0884     data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
0885     if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
0886         data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
0887     else
0888         data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
0889     data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
0890     data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
0891     WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
0892 
0893     data = RREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL);
0894     data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
0895         | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
0896         | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
0897         | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
0898         | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
0899         | UVD_CGC_CTRL__SYS_MODE_MASK
0900         | UVD_CGC_CTRL__UDEC_MODE_MASK
0901         | UVD_CGC_CTRL__MPEG2_MODE_MASK
0902         | UVD_CGC_CTRL__REGS_MODE_MASK
0903         | UVD_CGC_CTRL__RBC_MODE_MASK
0904         | UVD_CGC_CTRL__LMI_MC_MODE_MASK
0905         | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
0906         | UVD_CGC_CTRL__IDCT_MODE_MASK
0907         | UVD_CGC_CTRL__MPRD_MODE_MASK
0908         | UVD_CGC_CTRL__MPC_MODE_MASK
0909         | UVD_CGC_CTRL__LBSI_MODE_MASK
0910         | UVD_CGC_CTRL__LRBBM_MODE_MASK
0911         | UVD_CGC_CTRL__WCB_MODE_MASK
0912         | UVD_CGC_CTRL__VCPU_MODE_MASK
0913         | UVD_CGC_CTRL__MMSCH_MODE_MASK);
0914     WREG32_SOC15(VCN, inst, mmUVD_CGC_CTRL, data);
0915 
0916     data = RREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL);
0917     data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
0918         | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
0919         | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
0920         | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
0921         | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
0922         | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
0923         | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
0924         | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
0925         | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK
0926         | UVD_SUVD_CGC_CTRL__EFC_MODE_MASK
0927         | UVD_SUVD_CGC_CTRL__SAOE_MODE_MASK
0928         | UVD_SUVD_CGC_CTRL__SMPA_MODE_MASK
0929         | UVD_SUVD_CGC_CTRL__MPBE0_MODE_MASK
0930         | UVD_SUVD_CGC_CTRL__MPBE1_MODE_MASK
0931         | UVD_SUVD_CGC_CTRL__SIT_AV1_MODE_MASK
0932         | UVD_SUVD_CGC_CTRL__SDB_AV1_MODE_MASK
0933         | UVD_SUVD_CGC_CTRL__MPC1_MODE_MASK
0934         | UVD_SUVD_CGC_CTRL__FBC_PCLK_MASK
0935         | UVD_SUVD_CGC_CTRL__FBC_CCLK_MASK);
0936     WREG32_SOC15(VCN, inst, mmUVD_SUVD_CGC_CTRL, data);
0937 }
0938 
0939 static int vcn_v3_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
0940 {
0941     volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
0942     struct amdgpu_ring *ring;
0943     uint32_t rb_bufsz, tmp;
0944 
0945     /* disable register anti-hang mechanism */
0946     WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
0947         ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
0948     /* enable dynamic power gating mode */
0949     tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
0950     tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
0951     tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
0952     WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
0953 
0954     if (indirect)
0955         adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
0956 
0957     /* enable clock gating */
0958     vcn_v3_0_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
0959 
0960     /* enable VCPU clock */
0961     tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
0962     tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
0963     tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
0964     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0965         VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
0966 
0967     /* disable master interupt */
0968     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0969         VCN, inst_idx, mmUVD_MASTINT_EN), 0, 0, indirect);
0970 
0971     /* setup mmUVD_LMI_CTRL */
0972     tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
0973         UVD_LMI_CTRL__REQ_MODE_MASK |
0974         UVD_LMI_CTRL__CRC_RESET_MASK |
0975         UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
0976         UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
0977         UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
0978         (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
0979         0x00100000L);
0980     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0981         VCN, inst_idx, mmUVD_LMI_CTRL), tmp, 0, indirect);
0982 
0983     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0984         VCN, inst_idx, mmUVD_MPC_CNTL),
0985         0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
0986 
0987     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0988         VCN, inst_idx, mmUVD_MPC_SET_MUXA0),
0989         ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
0990          (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
0991          (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
0992          (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
0993 
0994     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
0995         VCN, inst_idx, mmUVD_MPC_SET_MUXB0),
0996          ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
0997          (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
0998          (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
0999          (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
1000 
1001     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1002         VCN, inst_idx, mmUVD_MPC_SET_MUX),
1003         ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1004          (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1005          (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
1006 
1007     vcn_v3_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
1008 
1009     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1010         VCN, inst_idx, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
1011     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1012         VCN, inst_idx, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
1013 
1014     /* enable LMI MC and UMC channels */
1015     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1016         VCN, inst_idx, mmUVD_LMI_CTRL2), 0, 0, indirect);
1017 
1018     /* unblock VCPU register access */
1019     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1020         VCN, inst_idx, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
1021 
1022     tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
1023     tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
1024     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1025         VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1026 
1027     /* enable master interrupt */
1028     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1029         VCN, inst_idx, mmUVD_MASTINT_EN),
1030         UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
1031 
1032     /* add nop to workaround PSP size check */
1033     WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
1034         VCN, inst_idx, mmUVD_VCPU_CNTL), tmp, 0, indirect);
1035 
1036     if (indirect)
1037         psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
1038             (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
1039                 (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
1040 
1041     ring = &adev->vcn.inst[inst_idx].ring_dec;
1042     /* force RBC into idle state */
1043     rb_bufsz = order_base_2(ring->ring_size);
1044     tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1045     tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1046     tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1047     tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1048     tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1049     WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
1050 
1051     /* Stall DPG before WPTR/RPTR reset */
1052     WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1053         UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1054         ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1055     fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1056 
1057     /* set the write pointer delay */
1058     WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
1059 
1060     /* set the wb address */
1061     WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
1062         (upper_32_bits(ring->gpu_addr) >> 2));
1063 
1064     /* programm the RB_BASE for ring buffer */
1065     WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1066         lower_32_bits(ring->gpu_addr));
1067     WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1068         upper_32_bits(ring->gpu_addr));
1069 
1070     /* Initialize the ring buffer's read and write pointers */
1071     WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
1072 
1073     WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
1074 
1075     ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
1076     WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
1077         lower_32_bits(ring->wptr));
1078 
1079     /* Reset FW shared memory RBC WPTR/RPTR */
1080     fw_shared->rb.rptr = 0;
1081     fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1082 
1083     /*resetting done, fw can check RB ring */
1084     fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1085 
1086     /* Unstall DPG */
1087     WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1088         0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1089 
1090     return 0;
1091 }
1092 
1093 static int vcn_v3_0_start(struct amdgpu_device *adev)
1094 {
1095     volatile struct amdgpu_fw_shared *fw_shared;
1096     struct amdgpu_ring *ring;
1097     uint32_t rb_bufsz, tmp;
1098     int i, j, k, r;
1099 
1100     if (adev->pm.dpm_enabled)
1101         amdgpu_dpm_enable_uvd(adev, true);
1102 
1103     for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1104         if (adev->vcn.harvest_config & (1 << i))
1105             continue;
1106 
1107         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG){
1108             r = vcn_v3_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1109             continue;
1110         }
1111 
1112         /* disable VCN power gating */
1113         vcn_v3_0_disable_static_power_gating(adev, i);
1114 
1115         /* set VCN status busy */
1116         tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1117         WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
1118 
1119         /*SW clock gating */
1120         vcn_v3_0_disable_clock_gating(adev, i);
1121 
1122         /* enable VCPU clock */
1123         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1124             UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1125 
1126         /* disable master interrupt */
1127         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
1128             ~UVD_MASTINT_EN__VCPU_EN_MASK);
1129 
1130         /* enable LMI MC and UMC channels */
1131         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1132             ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1133 
1134         tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1135         tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1136         tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1137         WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1138 
1139         /* setup mmUVD_LMI_CTRL */
1140         tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
1141         WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp |
1142             UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1143             UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1144             UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1145             UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1146 
1147         /* setup mmUVD_MPC_CNTL */
1148         tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
1149         tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1150         tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1151         WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
1152 
1153         /* setup UVD_MPC_SET_MUXA0 */
1154         WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
1155             ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1156             (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1157             (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1158             (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1159 
1160         /* setup UVD_MPC_SET_MUXB0 */
1161         WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
1162             ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1163             (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1164             (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1165             (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1166 
1167         /* setup mmUVD_MPC_SET_MUX */
1168         WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
1169             ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1170             (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1171             (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1172 
1173         vcn_v3_0_mc_resume(adev, i);
1174 
1175         /* VCN global tiling registers */
1176         WREG32_SOC15(VCN, i, mmUVD_GFX10_ADDR_CONFIG,
1177             adev->gfx.config.gb_addr_config);
1178 
1179         /* unblock VCPU register access */
1180         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1181             ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1182 
1183         /* release VCPU reset to boot */
1184         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1185             ~UVD_VCPU_CNTL__BLK_RST_MASK);
1186 
1187         for (j = 0; j < 10; ++j) {
1188             uint32_t status;
1189 
1190             for (k = 0; k < 100; ++k) {
1191                 status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1192                 if (status & 2)
1193                     break;
1194                 mdelay(10);
1195             }
1196             r = 0;
1197             if (status & 2)
1198                 break;
1199 
1200             DRM_ERROR("VCN[%d] decode not responding, trying to reset the VCPU!!!\n", i);
1201             WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1202                 UVD_VCPU_CNTL__BLK_RST_MASK,
1203                 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1204             mdelay(10);
1205             WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1206                 ~UVD_VCPU_CNTL__BLK_RST_MASK);
1207 
1208             mdelay(10);
1209             r = -1;
1210         }
1211 
1212         if (r) {
1213             DRM_ERROR("VCN[%d] decode not responding, giving up!!!\n", i);
1214             return r;
1215         }
1216 
1217         /* enable master interrupt */
1218         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1219             UVD_MASTINT_EN__VCPU_EN_MASK,
1220             ~UVD_MASTINT_EN__VCPU_EN_MASK);
1221 
1222         /* clear the busy bit of VCN_STATUS */
1223         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1224             ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1225 
1226         WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1227 
1228         ring = &adev->vcn.inst[i].ring_dec;
1229         /* force RBC into idle state */
1230         rb_bufsz = order_base_2(ring->ring_size);
1231         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1232         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1233         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1234         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1235         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1236         WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1237 
1238         fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1239         fw_shared->multi_queue.decode_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1240 
1241         /* programm the RB_BASE for ring buffer */
1242         WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1243             lower_32_bits(ring->gpu_addr));
1244         WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1245             upper_32_bits(ring->gpu_addr));
1246 
1247         /* Initialize the ring buffer's read and write pointers */
1248         WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1249 
1250         WREG32_SOC15(VCN, i, mmUVD_SCRATCH2, 0);
1251         ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1252         WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1253             lower_32_bits(ring->wptr));
1254         fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1255         fw_shared->multi_queue.decode_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1256 
1257         if (adev->ip_versions[UVD_HWIP][0] != IP_VERSION(3, 0, 33)) {
1258             fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1259             ring = &adev->vcn.inst[i].ring_enc[0];
1260             WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1261             WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1262             WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1263             WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1264             WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1265             fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1266 
1267             fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1268             ring = &adev->vcn.inst[i].ring_enc[1];
1269             WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1270             WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1271             WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1272             WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1273             WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1274             fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1275         }
1276     }
1277 
1278     return 0;
1279 }
1280 
1281 static int vcn_v3_0_start_sriov(struct amdgpu_device *adev)
1282 {
1283     int i, j;
1284     struct amdgpu_ring *ring;
1285     uint64_t cache_addr;
1286     uint64_t rb_addr;
1287     uint64_t ctx_addr;
1288     uint32_t param, resp, expected;
1289     uint32_t offset, cache_size;
1290     uint32_t tmp, timeout;
1291 
1292     struct amdgpu_mm_table *table = &adev->virt.mm_table;
1293     uint32_t *table_loc;
1294     uint32_t table_size;
1295     uint32_t size, size_dw;
1296 
1297     struct mmsch_v3_0_cmd_direct_write
1298         direct_wt = { {0} };
1299     struct mmsch_v3_0_cmd_direct_read_modify_write
1300         direct_rd_mod_wt = { {0} };
1301     struct mmsch_v3_0_cmd_end end = { {0} };
1302     struct mmsch_v3_0_init_header header;
1303 
1304     direct_wt.cmd_header.command_type =
1305         MMSCH_COMMAND__DIRECT_REG_WRITE;
1306     direct_rd_mod_wt.cmd_header.command_type =
1307         MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1308     end.cmd_header.command_type =
1309         MMSCH_COMMAND__END;
1310 
1311     header.version = MMSCH_VERSION;
1312     header.total_size = sizeof(struct mmsch_v3_0_init_header) >> 2;
1313     for (i = 0; i < AMDGPU_MAX_VCN_INSTANCES; i++) {
1314         header.inst[i].init_status = 0;
1315         header.inst[i].table_offset = 0;
1316         header.inst[i].table_size = 0;
1317     }
1318 
1319     table_loc = (uint32_t *)table->cpu_addr;
1320     table_loc += header.total_size;
1321     for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1322         if (adev->vcn.harvest_config & (1 << i))
1323             continue;
1324 
1325         table_size = 0;
1326 
1327         MMSCH_V3_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1328             mmUVD_STATUS),
1329             ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1330 
1331         cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1332 
1333         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1334             MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1335                 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1336                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1337             MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1338                 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1339                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1340             offset = 0;
1341             MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1342                 mmUVD_VCPU_CACHE_OFFSET0),
1343                 0);
1344         } else {
1345             MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1346                 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1347                 lower_32_bits(adev->vcn.inst[i].gpu_addr));
1348             MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1349                 mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1350                 upper_32_bits(adev->vcn.inst[i].gpu_addr));
1351             offset = cache_size;
1352             MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1353                 mmUVD_VCPU_CACHE_OFFSET0),
1354                 AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1355         }
1356 
1357         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1358             mmUVD_VCPU_CACHE_SIZE0),
1359             cache_size);
1360 
1361         cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1362         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1363             mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1364             lower_32_bits(cache_addr));
1365         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1366             mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1367             upper_32_bits(cache_addr));
1368         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1369             mmUVD_VCPU_CACHE_OFFSET1),
1370             0);
1371         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1372             mmUVD_VCPU_CACHE_SIZE1),
1373             AMDGPU_VCN_STACK_SIZE);
1374 
1375         cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1376             AMDGPU_VCN_STACK_SIZE;
1377         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1378             mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1379             lower_32_bits(cache_addr));
1380         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1381             mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1382             upper_32_bits(cache_addr));
1383         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1384             mmUVD_VCPU_CACHE_OFFSET2),
1385             0);
1386         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1387             mmUVD_VCPU_CACHE_SIZE2),
1388             AMDGPU_VCN_CONTEXT_SIZE);
1389 
1390         for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
1391             ring = &adev->vcn.inst[i].ring_enc[j];
1392             ring->wptr = 0;
1393             rb_addr = ring->gpu_addr;
1394             MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1395                 mmUVD_RB_BASE_LO),
1396                 lower_32_bits(rb_addr));
1397             MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1398                 mmUVD_RB_BASE_HI),
1399                 upper_32_bits(rb_addr));
1400             MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1401                 mmUVD_RB_SIZE),
1402                 ring->ring_size / 4);
1403         }
1404 
1405         ring = &adev->vcn.inst[i].ring_dec;
1406         ring->wptr = 0;
1407         rb_addr = ring->gpu_addr;
1408         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1409             mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1410             lower_32_bits(rb_addr));
1411         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1412             mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1413             upper_32_bits(rb_addr));
1414         /* force RBC into idle state */
1415         tmp = order_base_2(ring->ring_size);
1416         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, tmp);
1417         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1418         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1419         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1420         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1421         MMSCH_V3_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1422             mmUVD_RBC_RB_CNTL),
1423             tmp);
1424 
1425         /* add end packet */
1426         MMSCH_V3_0_INSERT_END();
1427 
1428         /* refine header */
1429         header.inst[i].init_status = 0;
1430         header.inst[i].table_offset = header.total_size;
1431         header.inst[i].table_size = table_size;
1432         header.total_size += table_size;
1433     }
1434 
1435     /* Update init table header in memory */
1436     size = sizeof(struct mmsch_v3_0_init_header);
1437     table_loc = (uint32_t *)table->cpu_addr;
1438     memcpy((void *)table_loc, &header, size);
1439 
1440     /* message MMSCH (in VCN[0]) to initialize this client
1441      * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1442      * of memory descriptor location
1443      */
1444     ctx_addr = table->gpu_addr;
1445     WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1446     WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1447 
1448     /* 2, update vmid of descriptor */
1449     tmp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1450     tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1451     /* use domain0 for MM scheduler */
1452     tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1453     WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, tmp);
1454 
1455     /* 3, notify mmsch about the size of this descriptor */
1456     size = header.total_size;
1457     WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1458 
1459     /* 4, set resp to zero */
1460     WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1461 
1462     /* 5, kick off the initialization and wait until
1463      * MMSCH_VF_MAILBOX_RESP becomes non-zero
1464      */
1465     param = 0x10000001;
1466     WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, param);
1467     tmp = 0;
1468     timeout = 1000;
1469     resp = 0;
1470     expected = param + 1;
1471     while (resp != expected) {
1472         resp = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1473         if (resp == expected)
1474             break;
1475 
1476         udelay(10);
1477         tmp = tmp + 10;
1478         if (tmp >= timeout) {
1479             DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1480                 " waiting for mmMMSCH_VF_MAILBOX_RESP "\
1481                 "(expected=0x%08x, readback=0x%08x)\n",
1482                 tmp, expected, resp);
1483             return -EBUSY;
1484         }
1485     }
1486 
1487     return 0;
1488 }
1489 
1490 static int vcn_v3_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1491 {
1492     struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE};
1493     uint32_t tmp;
1494 
1495     vcn_v3_0_pause_dpg_mode(adev, inst_idx, &state);
1496 
1497     /* Wait for power status to be 1 */
1498     SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1499         UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1500 
1501     /* wait for read ptr to be equal to write ptr */
1502     tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1503     SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1504 
1505     tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1506     SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1507 
1508     tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1509     SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1510 
1511     SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1512         UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1513 
1514     /* disable dynamic power gating mode */
1515     WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1516         ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1517 
1518     return 0;
1519 }
1520 
1521 static int vcn_v3_0_stop(struct amdgpu_device *adev)
1522 {
1523     uint32_t tmp;
1524     int i, r = 0;
1525 
1526     for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1527         if (adev->vcn.harvest_config & (1 << i))
1528             continue;
1529 
1530         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1531             r = vcn_v3_0_stop_dpg_mode(adev, i);
1532             continue;
1533         }
1534 
1535         /* wait for vcn idle */
1536         r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1537         if (r)
1538             return r;
1539 
1540         tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1541             UVD_LMI_STATUS__READ_CLEAN_MASK |
1542             UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1543             UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1544         r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1545         if (r)
1546             return r;
1547 
1548         /* disable LMI UMC channel */
1549         tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1550         tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1551         WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1552         tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1553             UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1554         r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1555         if (r)
1556             return r;
1557 
1558         /* block VCPU register access */
1559         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1560             UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1561             ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1562 
1563         /* reset VCPU */
1564         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1565             UVD_VCPU_CNTL__BLK_RST_MASK,
1566             ~UVD_VCPU_CNTL__BLK_RST_MASK);
1567 
1568         /* disable VCPU clock */
1569         WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1570             ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1571 
1572         /* apply soft reset */
1573         tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1574         tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1575         WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1576         tmp = RREG32_SOC15(VCN, i, mmUVD_SOFT_RESET);
1577         tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1578         WREG32_SOC15(VCN, i, mmUVD_SOFT_RESET, tmp);
1579 
1580         /* clear status */
1581         WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1582 
1583         /* apply HW clock gating */
1584         vcn_v3_0_enable_clock_gating(adev, i);
1585 
1586         /* enable VCN power gating */
1587         vcn_v3_0_enable_static_power_gating(adev, i);
1588     }
1589 
1590     if (adev->pm.dpm_enabled)
1591         amdgpu_dpm_enable_uvd(adev, false);
1592 
1593     return 0;
1594 }
1595 
1596 static int vcn_v3_0_pause_dpg_mode(struct amdgpu_device *adev,
1597            int inst_idx, struct dpg_pause_state *new_state)
1598 {
1599     volatile struct amdgpu_fw_shared *fw_shared;
1600     struct amdgpu_ring *ring;
1601     uint32_t reg_data = 0;
1602     int ret_code;
1603 
1604     /* pause/unpause if state is changed */
1605     if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1606         DRM_DEBUG("dpg pause state changed %d -> %d",
1607             adev->vcn.inst[inst_idx].pause_state.fw_based,  new_state->fw_based);
1608         reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1609             (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1610 
1611         if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1612             ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1613                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1614 
1615             if (!ret_code) {
1616                 /* pause DPG */
1617                 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1618                 WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1619 
1620                 /* wait for ACK */
1621                 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1622                     UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1623                     UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1624 
1625                 /* Stall DPG before WPTR/RPTR reset */
1626                 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1627                     UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1628                     ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1629 
1630                 if (adev->ip_versions[UVD_HWIP][0] != IP_VERSION(3, 0, 33)) {
1631                     /* Restore */
1632                     fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1633                     fw_shared->multi_queue.encode_generalpurpose_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1634                     ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1635                     ring->wptr = 0;
1636                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1637                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1638                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1639                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1640                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1641                     fw_shared->multi_queue.encode_generalpurpose_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1642 
1643                     fw_shared->multi_queue.encode_lowlatency_queue_mode |= cpu_to_le32(FW_QUEUE_RING_RESET);
1644                     ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1645                     ring->wptr = 0;
1646                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1647                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1648                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1649                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1650                     WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1651                     fw_shared->multi_queue.encode_lowlatency_queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET);
1652 
1653                     /* restore wptr/rptr with pointers saved in FW shared memory*/
1654                     WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, fw_shared->rb.rptr);
1655                     WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR, fw_shared->rb.wptr);
1656                 }
1657 
1658                 /* Unstall DPG */
1659                 WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1660                     0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1661 
1662                 SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1663                     UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1664             }
1665         } else {
1666             /* unpause dpg, no need to wait */
1667             reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1668             WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1669         }
1670         adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1671     }
1672 
1673     return 0;
1674 }
1675 
1676 /**
1677  * vcn_v3_0_dec_ring_get_rptr - get read pointer
1678  *
1679  * @ring: amdgpu_ring pointer
1680  *
1681  * Returns the current hardware read pointer
1682  */
1683 static uint64_t vcn_v3_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1684 {
1685     struct amdgpu_device *adev = ring->adev;
1686 
1687     return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1688 }
1689 
1690 /**
1691  * vcn_v3_0_dec_ring_get_wptr - get write pointer
1692  *
1693  * @ring: amdgpu_ring pointer
1694  *
1695  * Returns the current hardware write pointer
1696  */
1697 static uint64_t vcn_v3_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1698 {
1699     struct amdgpu_device *adev = ring->adev;
1700 
1701     if (ring->use_doorbell)
1702         return *ring->wptr_cpu_addr;
1703     else
1704         return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1705 }
1706 
1707 /**
1708  * vcn_v3_0_dec_ring_set_wptr - set write pointer
1709  *
1710  * @ring: amdgpu_ring pointer
1711  *
1712  * Commits the write pointer to the hardware
1713  */
1714 static void vcn_v3_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1715 {
1716     struct amdgpu_device *adev = ring->adev;
1717     volatile struct amdgpu_fw_shared *fw_shared;
1718 
1719     if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1720         /*whenever update RBC_RB_WPTR, we save the wptr in shared rb.wptr and scratch2 */
1721         fw_shared = adev->vcn.inst[ring->me].fw_shared.cpu_addr;
1722         fw_shared->rb.wptr = lower_32_bits(ring->wptr);
1723         WREG32_SOC15(VCN, ring->me, mmUVD_SCRATCH2,
1724             lower_32_bits(ring->wptr));
1725     }
1726 
1727     if (ring->use_doorbell) {
1728         *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1729         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1730     } else {
1731         WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1732     }
1733 }
1734 
1735 static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
1736     .type = AMDGPU_RING_TYPE_VCN_DEC,
1737     .align_mask = 0x3f,
1738     .nop = VCN_DEC_SW_CMD_NO_OP,
1739     .secure_submission_supported = true,
1740     .vmhub = AMDGPU_MMHUB_0,
1741     .get_rptr = vcn_v3_0_dec_ring_get_rptr,
1742     .get_wptr = vcn_v3_0_dec_ring_get_wptr,
1743     .set_wptr = vcn_v3_0_dec_ring_set_wptr,
1744     .emit_frame_size =
1745         SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1746         SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1747         VCN_SW_RING_EMIT_FRAME_SIZE,
1748     .emit_ib_size = 5, /* vcn_dec_sw_ring_emit_ib */
1749     .emit_ib = vcn_dec_sw_ring_emit_ib,
1750     .emit_fence = vcn_dec_sw_ring_emit_fence,
1751     .emit_vm_flush = vcn_dec_sw_ring_emit_vm_flush,
1752     .test_ring = amdgpu_vcn_dec_sw_ring_test_ring,
1753     .test_ib = NULL,//amdgpu_vcn_dec_sw_ring_test_ib,
1754     .insert_nop = amdgpu_ring_insert_nop,
1755     .insert_end = vcn_dec_sw_ring_insert_end,
1756     .pad_ib = amdgpu_ring_generic_pad_ib,
1757     .begin_use = amdgpu_vcn_ring_begin_use,
1758     .end_use = amdgpu_vcn_ring_end_use,
1759     .emit_wreg = vcn_dec_sw_ring_emit_wreg,
1760     .emit_reg_wait = vcn_dec_sw_ring_emit_reg_wait,
1761     .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1762 };
1763 
1764 static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p)
1765 {
1766     struct drm_gpu_scheduler **scheds;
1767 
1768     /* The create msg must be in the first IB submitted */
1769     if (atomic_read(&p->entity->fence_seq))
1770         return -EINVAL;
1771 
1772     scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC]
1773         [AMDGPU_RING_PRIO_DEFAULT].sched;
1774     drm_sched_entity_modify_sched(p->entity, scheds, 1);
1775     return 0;
1776 }
1777 
1778 static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr)
1779 {
1780     struct ttm_operation_ctx ctx = { false, false };
1781     struct amdgpu_bo_va_mapping *map;
1782     uint32_t *msg, num_buffers;
1783     struct amdgpu_bo *bo;
1784     uint64_t start, end;
1785     unsigned int i;
1786     void * ptr;
1787     int r;
1788 
1789     addr &= AMDGPU_GMC_HOLE_MASK;
1790     r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1791     if (r) {
1792         DRM_ERROR("Can't find BO for addr 0x%08Lx\n", addr);
1793         return r;
1794     }
1795 
1796     start = map->start * AMDGPU_GPU_PAGE_SIZE;
1797     end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1798     if (addr & 0x7) {
1799         DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1800         return -EINVAL;
1801     }
1802 
1803     bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1804     amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1805     r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1806     if (r) {
1807         DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1808         return r;
1809     }
1810 
1811     r = amdgpu_bo_kmap(bo, &ptr);
1812     if (r) {
1813         DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1814         return r;
1815     }
1816 
1817     msg = ptr + addr - start;
1818 
1819     /* Check length */
1820     if (msg[1] > end - addr) {
1821         r = -EINVAL;
1822         goto out;
1823     }
1824 
1825     if (msg[3] != RDECODE_MSG_CREATE)
1826         goto out;
1827 
1828     num_buffers = msg[2];
1829     for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1830         uint32_t offset, size, *create;
1831 
1832         if (msg[0] != RDECODE_MESSAGE_CREATE)
1833             continue;
1834 
1835         offset = msg[1];
1836         size = msg[2];
1837 
1838         if (offset + size > end) {
1839             r = -EINVAL;
1840             goto out;
1841         }
1842 
1843         create = ptr + addr + offset - start;
1844 
1845         /* H246, HEVC and VP9 can run on any instance */
1846         if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1847             continue;
1848 
1849         r = vcn_v3_0_limit_sched(p);
1850         if (r)
1851             goto out;
1852     }
1853 
1854 out:
1855     amdgpu_bo_kunmap(bo);
1856     return r;
1857 }
1858 
1859 static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1860                        struct amdgpu_job *job,
1861                        struct amdgpu_ib *ib)
1862 {
1863     struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched);
1864     uint32_t msg_lo = 0, msg_hi = 0;
1865     unsigned i;
1866     int r;
1867 
1868     /* The first instance can decode anything */
1869     if (!ring->me)
1870         return 0;
1871 
1872     for (i = 0; i < ib->length_dw; i += 2) {
1873         uint32_t reg = amdgpu_ib_get_value(ib, i);
1874         uint32_t val = amdgpu_ib_get_value(ib, i + 1);
1875 
1876         if (reg == PACKET0(p->adev->vcn.internal.data0, 0)) {
1877             msg_lo = val;
1878         } else if (reg == PACKET0(p->adev->vcn.internal.data1, 0)) {
1879             msg_hi = val;
1880         } else if (reg == PACKET0(p->adev->vcn.internal.cmd, 0) &&
1881                val == 0) {
1882             r = vcn_v3_0_dec_msg(p, ((u64)msg_hi) << 32 | msg_lo);
1883             if (r)
1884                 return r;
1885         }
1886     }
1887     return 0;
1888 }
1889 
1890 static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = {
1891     .type = AMDGPU_RING_TYPE_VCN_DEC,
1892     .align_mask = 0xf,
1893     .secure_submission_supported = true,
1894     .vmhub = AMDGPU_MMHUB_0,
1895     .get_rptr = vcn_v3_0_dec_ring_get_rptr,
1896     .get_wptr = vcn_v3_0_dec_ring_get_wptr,
1897     .set_wptr = vcn_v3_0_dec_ring_set_wptr,
1898     .patch_cs_in_place = vcn_v3_0_ring_patch_cs_in_place,
1899     .emit_frame_size =
1900         SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1901         SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1902         8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1903         14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1904         6,
1905     .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1906     .emit_ib = vcn_v2_0_dec_ring_emit_ib,
1907     .emit_fence = vcn_v2_0_dec_ring_emit_fence,
1908     .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1909     .test_ring = vcn_v2_0_dec_ring_test_ring,
1910     .test_ib = amdgpu_vcn_dec_ring_test_ib,
1911     .insert_nop = vcn_v2_0_dec_ring_insert_nop,
1912     .insert_start = vcn_v2_0_dec_ring_insert_start,
1913     .insert_end = vcn_v2_0_dec_ring_insert_end,
1914     .pad_ib = amdgpu_ring_generic_pad_ib,
1915     .begin_use = amdgpu_vcn_ring_begin_use,
1916     .end_use = amdgpu_vcn_ring_end_use,
1917     .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1918     .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1919     .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1920 };
1921 
1922 /**
1923  * vcn_v3_0_enc_ring_get_rptr - get enc read pointer
1924  *
1925  * @ring: amdgpu_ring pointer
1926  *
1927  * Returns the current hardware enc read pointer
1928  */
1929 static uint64_t vcn_v3_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1930 {
1931     struct amdgpu_device *adev = ring->adev;
1932 
1933     if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1934         return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1935     else
1936         return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1937 }
1938 
1939 /**
1940  * vcn_v3_0_enc_ring_get_wptr - get enc write pointer
1941  *
1942  * @ring: amdgpu_ring pointer
1943  *
1944  * Returns the current hardware enc write pointer
1945  */
1946 static uint64_t vcn_v3_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1947 {
1948     struct amdgpu_device *adev = ring->adev;
1949 
1950     if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1951         if (ring->use_doorbell)
1952             return *ring->wptr_cpu_addr;
1953         else
1954             return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1955     } else {
1956         if (ring->use_doorbell)
1957             return *ring->wptr_cpu_addr;
1958         else
1959             return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1960     }
1961 }
1962 
1963 /**
1964  * vcn_v3_0_enc_ring_set_wptr - set enc write pointer
1965  *
1966  * @ring: amdgpu_ring pointer
1967  *
1968  * Commits the enc write pointer to the hardware
1969  */
1970 static void vcn_v3_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1971 {
1972     struct amdgpu_device *adev = ring->adev;
1973 
1974     if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1975         if (ring->use_doorbell) {
1976             *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1977             WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1978         } else {
1979             WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1980         }
1981     } else {
1982         if (ring->use_doorbell) {
1983             *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1984             WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1985         } else {
1986             WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1987         }
1988     }
1989 }
1990 
1991 static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = {
1992     .type = AMDGPU_RING_TYPE_VCN_ENC,
1993     .align_mask = 0x3f,
1994     .nop = VCN_ENC_CMD_NO_OP,
1995     .vmhub = AMDGPU_MMHUB_0,
1996     .get_rptr = vcn_v3_0_enc_ring_get_rptr,
1997     .get_wptr = vcn_v3_0_enc_ring_get_wptr,
1998     .set_wptr = vcn_v3_0_enc_ring_set_wptr,
1999     .emit_frame_size =
2000         SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2001         SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2002         4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
2003         5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
2004         1, /* vcn_v2_0_enc_ring_insert_end */
2005     .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
2006     .emit_ib = vcn_v2_0_enc_ring_emit_ib,
2007     .emit_fence = vcn_v2_0_enc_ring_emit_fence,
2008     .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
2009     .test_ring = amdgpu_vcn_enc_ring_test_ring,
2010     .test_ib = amdgpu_vcn_enc_ring_test_ib,
2011     .insert_nop = amdgpu_ring_insert_nop,
2012     .insert_end = vcn_v2_0_enc_ring_insert_end,
2013     .pad_ib = amdgpu_ring_generic_pad_ib,
2014     .begin_use = amdgpu_vcn_ring_begin_use,
2015     .end_use = amdgpu_vcn_ring_end_use,
2016     .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
2017     .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
2018     .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2019 };
2020 
2021 static void vcn_v3_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2022 {
2023     int i;
2024 
2025     for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2026         if (adev->vcn.harvest_config & (1 << i))
2027             continue;
2028 
2029         if (!DEC_SW_RING_ENABLED)
2030             adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_ring_vm_funcs;
2031         else
2032             adev->vcn.inst[i].ring_dec.funcs = &vcn_v3_0_dec_sw_ring_vm_funcs;
2033         adev->vcn.inst[i].ring_dec.me = i;
2034         DRM_INFO("VCN(%d) decode%s is enabled in VM mode\n", i,
2035               DEC_SW_RING_ENABLED?"(Software Ring)":"");
2036     }
2037 }
2038 
2039 static void vcn_v3_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2040 {
2041     int i, j;
2042 
2043     for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2044         if (adev->vcn.harvest_config & (1 << i))
2045             continue;
2046 
2047         for (j = 0; j < adev->vcn.num_enc_rings; ++j) {
2048             adev->vcn.inst[i].ring_enc[j].funcs = &vcn_v3_0_enc_ring_vm_funcs;
2049             adev->vcn.inst[i].ring_enc[j].me = i;
2050         }
2051         if (adev->vcn.num_enc_rings > 0)
2052             DRM_INFO("VCN(%d) encode is enabled in VM mode\n", i);
2053     }
2054 }
2055 
2056 static bool vcn_v3_0_is_idle(void *handle)
2057 {
2058     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2059     int i, ret = 1;
2060 
2061     for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2062         if (adev->vcn.harvest_config & (1 << i))
2063             continue;
2064 
2065         ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
2066     }
2067 
2068     return ret;
2069 }
2070 
2071 static int vcn_v3_0_wait_for_idle(void *handle)
2072 {
2073     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2074     int i, ret = 0;
2075 
2076     for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2077         if (adev->vcn.harvest_config & (1 << i))
2078             continue;
2079 
2080         ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
2081             UVD_STATUS__IDLE);
2082         if (ret)
2083             return ret;
2084     }
2085 
2086     return ret;
2087 }
2088 
2089 static int vcn_v3_0_set_clockgating_state(void *handle,
2090                       enum amd_clockgating_state state)
2091 {
2092     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2093     bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
2094     int i;
2095 
2096     for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2097         if (adev->vcn.harvest_config & (1 << i))
2098             continue;
2099 
2100         if (enable) {
2101             if (RREG32_SOC15(VCN, i, mmUVD_STATUS) != UVD_STATUS__IDLE)
2102                 return -EBUSY;
2103             vcn_v3_0_enable_clock_gating(adev, i);
2104         } else {
2105             vcn_v3_0_disable_clock_gating(adev, i);
2106         }
2107     }
2108 
2109     return 0;
2110 }
2111 
2112 static int vcn_v3_0_set_powergating_state(void *handle,
2113                       enum amd_powergating_state state)
2114 {
2115     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2116     int ret;
2117 
2118     /* for SRIOV, guest should not control VCN Power-gating
2119      * MMSCH FW should control Power-gating and clock-gating
2120      * guest should avoid touching CGC and PG
2121      */
2122     if (amdgpu_sriov_vf(adev)) {
2123         adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
2124         return 0;
2125     }
2126 
2127     if(state == adev->vcn.cur_state)
2128         return 0;
2129 
2130     if (state == AMD_PG_STATE_GATE)
2131         ret = vcn_v3_0_stop(adev);
2132     else
2133         ret = vcn_v3_0_start(adev);
2134 
2135     if(!ret)
2136         adev->vcn.cur_state = state;
2137 
2138     return ret;
2139 }
2140 
2141 static int vcn_v3_0_set_interrupt_state(struct amdgpu_device *adev,
2142                     struct amdgpu_irq_src *source,
2143                     unsigned type,
2144                     enum amdgpu_interrupt_state state)
2145 {
2146     return 0;
2147 }
2148 
2149 static int vcn_v3_0_process_interrupt(struct amdgpu_device *adev,
2150                       struct amdgpu_irq_src *source,
2151                       struct amdgpu_iv_entry *entry)
2152 {
2153     uint32_t ip_instance;
2154 
2155     switch (entry->client_id) {
2156     case SOC15_IH_CLIENTID_VCN:
2157         ip_instance = 0;
2158         break;
2159     case SOC15_IH_CLIENTID_VCN1:
2160         ip_instance = 1;
2161         break;
2162     default:
2163         DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
2164         return 0;
2165     }
2166 
2167     DRM_DEBUG("IH: VCN TRAP\n");
2168 
2169     switch (entry->src_id) {
2170     case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
2171         amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
2172         break;
2173     case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
2174         amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
2175         break;
2176     case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
2177         amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
2178         break;
2179     default:
2180         DRM_ERROR("Unhandled interrupt: %d %d\n",
2181               entry->src_id, entry->src_data[0]);
2182         break;
2183     }
2184 
2185     return 0;
2186 }
2187 
2188 static const struct amdgpu_irq_src_funcs vcn_v3_0_irq_funcs = {
2189     .set = vcn_v3_0_set_interrupt_state,
2190     .process = vcn_v3_0_process_interrupt,
2191 };
2192 
2193 static void vcn_v3_0_set_irq_funcs(struct amdgpu_device *adev)
2194 {
2195     int i;
2196 
2197     for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
2198         if (adev->vcn.harvest_config & (1 << i))
2199             continue;
2200 
2201         adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
2202         adev->vcn.inst[i].irq.funcs = &vcn_v3_0_irq_funcs;
2203     }
2204 }
2205 
2206 static const struct amd_ip_funcs vcn_v3_0_ip_funcs = {
2207     .name = "vcn_v3_0",
2208     .early_init = vcn_v3_0_early_init,
2209     .late_init = NULL,
2210     .sw_init = vcn_v3_0_sw_init,
2211     .sw_fini = vcn_v3_0_sw_fini,
2212     .hw_init = vcn_v3_0_hw_init,
2213     .hw_fini = vcn_v3_0_hw_fini,
2214     .suspend = vcn_v3_0_suspend,
2215     .resume = vcn_v3_0_resume,
2216     .is_idle = vcn_v3_0_is_idle,
2217     .wait_for_idle = vcn_v3_0_wait_for_idle,
2218     .check_soft_reset = NULL,
2219     .pre_soft_reset = NULL,
2220     .soft_reset = NULL,
2221     .post_soft_reset = NULL,
2222     .set_clockgating_state = vcn_v3_0_set_clockgating_state,
2223     .set_powergating_state = vcn_v3_0_set_powergating_state,
2224 };
2225 
2226 const struct amdgpu_ip_block_version vcn_v3_0_ip_block =
2227 {
2228     .type = AMD_IP_BLOCK_TYPE_VCN,
2229     .major = 3,
2230     .minor = 0,
2231     .rev = 0,
2232     .funcs = &vcn_v3_0_ip_funcs,
2233 };
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!