amd/amdgpu/cz_ih.c

0001 /*
0002  * Copyright 2014 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023
0024 #include <linux/pci.h>
0025
0026 #include "amdgpu.h"
0027 #include "amdgpu_ih.h"
0028 #include "vid.h"
0029
0030 #include "oss/oss_3_0_1_d.h"
0031 #include "oss/oss_3_0_1_sh_mask.h"
0032
0033 #include "bif/bif_5_1_d.h"
0034 #include "bif/bif_5_1_sh_mask.h"
0035
0036 /*
0037  * Interrupts
0038  * Starting with r6xx, interrupts are handled via a ring buffer.
0039  * Ring buffers are areas of GPU accessible memory that the GPU
0040  * writes interrupt vectors into and the host reads vectors out of.
0041  * There is a rptr (read pointer) that determines where the
0042  * host is currently reading, and a wptr (write pointer)
0043  * which determines where the GPU has written.  When the
0044  * pointers are equal, the ring is idle.  When the GPU
0045  * writes vectors to the ring buffer, it increments the
0046  * wptr.  When there is an interrupt, the host then starts
0047  * fetching commands and processing them until the pointers are
0048  * equal again at which point it updates the rptr.
0049  */
0050
0051 static void cz_ih_set_interrupt_funcs(struct amdgpu_device *adev);
0052
0053 /**
0054  * cz_ih_enable_interrupts - Enable the interrupt ring buffer
0055  *
0056  * @adev: amdgpu_device pointer
0057  *
0058  * Enable the interrupt ring buffer (VI).
0059  */
0060 static void cz_ih_enable_interrupts(struct amdgpu_device *adev)
0061 {
0062     u32 ih_cntl = RREG32(mmIH_CNTL);
0063     u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
0064
0065     ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 1);
0066     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
0067     WREG32(mmIH_CNTL, ih_cntl);
0068     WREG32(mmIH_RB_CNTL, ih_rb_cntl);
0069     adev->irq.ih.enabled = true;
0070 }
0071
0072 /**
0073  * cz_ih_disable_interrupts - Disable the interrupt ring buffer
0074  *
0075  * @adev: amdgpu_device pointer
0076  *
0077  * Disable the interrupt ring buffer (VI).
0078  */
0079 static void cz_ih_disable_interrupts(struct amdgpu_device *adev)
0080 {
0081     u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
0082     u32 ih_cntl = RREG32(mmIH_CNTL);
0083
0084     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
0085     ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 0);
0086     WREG32(mmIH_RB_CNTL, ih_rb_cntl);
0087     WREG32(mmIH_CNTL, ih_cntl);
0088     /* set rptr, wptr to 0 */
0089     WREG32(mmIH_RB_RPTR, 0);
0090     WREG32(mmIH_RB_WPTR, 0);
0091     adev->irq.ih.enabled = false;
0092     adev->irq.ih.rptr = 0;
0093 }
0094
0095 /**
0096  * cz_ih_irq_init - init and enable the interrupt ring
0097  *
0098  * @adev: amdgpu_device pointer
0099  *
0100  * Allocate a ring buffer for the interrupt controller,
0101  * enable the RLC, disable interrupts, enable the IH
0102  * ring buffer and enable it (VI).
0103  * Called at device load and reume.
0104  * Returns 0 for success, errors for failure.
0105  */
0106 static int cz_ih_irq_init(struct amdgpu_device *adev)
0107 {
0108     struct amdgpu_ih_ring *ih = &adev->irq.ih;
0109     u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
0110     int rb_bufsz;
0111
0112     /* disable irqs */
0113     cz_ih_disable_interrupts(adev);
0114
0115     /* setup interrupt control */
0116     WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
0117     interrupt_cntl = RREG32(mmINTERRUPT_CNTL);
0118     /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
0119      * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
0120      */
0121     interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0);
0122     /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
0123     interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0);
0124     WREG32(mmINTERRUPT_CNTL, interrupt_cntl);
0125
0126     /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
0127     WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
0128
0129     rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
0130     ih_rb_cntl = REG_SET_FIELD(0, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
0131     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
0132     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
0133
0134     /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
0135     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
0136
0137     /* set the writeback address whether it's enabled or not */
0138     WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr));
0139     WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF);
0140
0141     WREG32(mmIH_RB_CNTL, ih_rb_cntl);
0142
0143     /* set rptr, wptr to 0 */
0144     WREG32(mmIH_RB_RPTR, 0);
0145     WREG32(mmIH_RB_WPTR, 0);
0146
0147     /* Default settings for IH_CNTL (disabled at first) */
0148     ih_cntl = RREG32(mmIH_CNTL);
0149     ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, MC_VMID, 0);
0150
0151     if (adev->irq.msi_enabled)
0152         ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, RPTR_REARM, 1);
0153     WREG32(mmIH_CNTL, ih_cntl);
0154
0155     pci_set_master(adev->pdev);
0156
0157     /* enable interrupts */
0158     cz_ih_enable_interrupts(adev);
0159
0160     return 0;
0161 }
0162
0163 /**
0164  * cz_ih_irq_disable - disable interrupts
0165  *
0166  * @adev: amdgpu_device pointer
0167  *
0168  * Disable interrupts on the hw (VI).
0169  */
0170 static void cz_ih_irq_disable(struct amdgpu_device *adev)
0171 {
0172     cz_ih_disable_interrupts(adev);
0173
0174     /* Wait and acknowledge irq */
0175     mdelay(1);
0176 }
0177
0178 /**
0179  * cz_ih_get_wptr - get the IH ring buffer wptr
0180  *
0181  * @adev: amdgpu_device pointer
0182  * @ih: IH ring buffer to fetch wptr
0183  *
0184  * Get the IH ring buffer wptr from either the register
0185  * or the writeback memory buffer (VI).  Also check for
0186  * ring buffer overflow and deal with it.
0187  * Used by cz_irq_process(VI).
0188  * Returns the value of the wptr.
0189  */
0190 static u32 cz_ih_get_wptr(struct amdgpu_device *adev,
0191               struct amdgpu_ih_ring *ih)
0192 {
0193     u32 wptr, tmp;
0194
0195     wptr = le32_to_cpu(*ih->wptr_cpu);
0196
0197     if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
0198         goto out;
0199
0200     /* Double check that the overflow wasn't already cleared. */
0201     wptr = RREG32(mmIH_RB_WPTR);
0202
0203     if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
0204         goto out;
0205
0206     wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
0207
0208     /* When a ring buffer overflow happen start parsing interrupt
0209      * from the last not overwritten vector (wptr + 16). Hopefully
0210      * this should allow us to catchup.
0211      */
0212     dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
0213         wptr, ih->rptr, (wptr + 16) & ih->ptr_mask);
0214     ih->rptr = (wptr + 16) & ih->ptr_mask;
0215     tmp = RREG32(mmIH_RB_CNTL);
0216     tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
0217     WREG32(mmIH_RB_CNTL, tmp);
0218
0219
0220 out:
0221     return (wptr & ih->ptr_mask);
0222 }
0223
0224 /**
0225  * cz_ih_decode_iv - decode an interrupt vector
0226  *
0227  * @adev: amdgpu_device pointer
0228  * @ih: IH ring buffer to decode
0229  * @entry: IV entry to place decoded information into
0230  *
0231  * Decodes the interrupt vector at the current rptr
0232  * position and also advance the position.
0233  */
0234 static void cz_ih_decode_iv(struct amdgpu_device *adev,
0235                 struct amdgpu_ih_ring *ih,
0236                 struct amdgpu_iv_entry *entry)
0237 {
0238     /* wptr/rptr are in bytes! */
0239     u32 ring_index = ih->rptr >> 2;
0240     uint32_t dw[4];
0241
0242     dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
0243     dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
0244     dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
0245     dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
0246
0247     entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
0248     entry->src_id = dw[0] & 0xff;
0249     entry->src_data[0] = dw[1] & 0xfffffff;
0250     entry->ring_id = dw[2] & 0xff;
0251     entry->vmid = (dw[2] >> 8) & 0xff;
0252     entry->pasid = (dw[2] >> 16) & 0xffff;
0253
0254     /* wptr/rptr are in bytes! */
0255     ih->rptr += 16;
0256 }
0257
0258 /**
0259  * cz_ih_set_rptr - set the IH ring buffer rptr
0260  *
0261  * @adev: amdgpu_device pointer
0262  * @ih: IH ring buffer to set rptr
0263  *
0264  * Set the IH ring buffer rptr.
0265  */
0266 static void cz_ih_set_rptr(struct amdgpu_device *adev,
0267                struct amdgpu_ih_ring *ih)
0268 {
0269     WREG32(mmIH_RB_RPTR, ih->rptr);
0270 }
0271
0272 static int cz_ih_early_init(void *handle)
0273 {
0274     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0275     int ret;
0276
0277     ret = amdgpu_irq_add_domain(adev);
0278     if (ret)
0279         return ret;
0280
0281     cz_ih_set_interrupt_funcs(adev);
0282
0283     return 0;
0284 }
0285
0286 static int cz_ih_sw_init(void *handle)
0287 {
0288     int r;
0289     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0290
0291     r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, false);
0292     if (r)
0293         return r;
0294
0295     r = amdgpu_irq_init(adev);
0296
0297     return r;
0298 }
0299
0300 static int cz_ih_sw_fini(void *handle)
0301 {
0302     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0303
0304     amdgpu_irq_fini_sw(adev);
0305     amdgpu_irq_remove_domain(adev);
0306
0307     return 0;
0308 }
0309
0310 static int cz_ih_hw_init(void *handle)
0311 {
0312     int r;
0313     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0314
0315     r = cz_ih_irq_init(adev);
0316     if (r)
0317         return r;
0318
0319     return 0;
0320 }
0321
0322 static int cz_ih_hw_fini(void *handle)
0323 {
0324     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0325
0326     cz_ih_irq_disable(adev);
0327
0328     return 0;
0329 }
0330
0331 static int cz_ih_suspend(void *handle)
0332 {
0333     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0334
0335     return cz_ih_hw_fini(adev);
0336 }
0337
0338 static int cz_ih_resume(void *handle)
0339 {
0340     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0341
0342     return cz_ih_hw_init(adev);
0343 }
0344
0345 static bool cz_ih_is_idle(void *handle)
0346 {
0347     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0348     u32 tmp = RREG32(mmSRBM_STATUS);
0349
0350     if (REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))
0351         return false;
0352
0353     return true;
0354 }
0355
0356 static int cz_ih_wait_for_idle(void *handle)
0357 {
0358     unsigned i;
0359     u32 tmp;
0360     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0361
0362     for (i = 0; i < adev->usec_timeout; i++) {
0363         /* read MC_STATUS */
0364         tmp = RREG32(mmSRBM_STATUS);
0365         if (!REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))
0366             return 0;
0367         udelay(1);
0368     }
0369     return -ETIMEDOUT;
0370 }
0371
0372 static int cz_ih_soft_reset(void *handle)
0373 {
0374     u32 srbm_soft_reset = 0;
0375     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0376     u32 tmp = RREG32(mmSRBM_STATUS);
0377
0378     if (tmp & SRBM_STATUS__IH_BUSY_MASK)
0379         srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
0380                         SOFT_RESET_IH, 1);
0381
0382     if (srbm_soft_reset) {
0383         tmp = RREG32(mmSRBM_SOFT_RESET);
0384         tmp |= srbm_soft_reset;
0385         dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
0386         WREG32(mmSRBM_SOFT_RESET, tmp);
0387         tmp = RREG32(mmSRBM_SOFT_RESET);
0388
0389         udelay(50);
0390
0391         tmp &= ~srbm_soft_reset;
0392         WREG32(mmSRBM_SOFT_RESET, tmp);
0393         tmp = RREG32(mmSRBM_SOFT_RESET);
0394
0395         /* Wait a little for things to settle down */
0396         udelay(50);
0397     }
0398
0399     return 0;
0400 }
0401
0402 static int cz_ih_set_clockgating_state(void *handle,
0403                       enum amd_clockgating_state state)
0404 {
0405     // TODO
0406     return 0;
0407 }
0408
0409 static int cz_ih_set_powergating_state(void *handle,
0410                       enum amd_powergating_state state)
0411 {
0412     // TODO
0413     return 0;
0414 }
0415
0416 static const struct amd_ip_funcs cz_ih_ip_funcs = {
0417     .name = "cz_ih",
0418     .early_init = cz_ih_early_init,
0419     .late_init = NULL,
0420     .sw_init = cz_ih_sw_init,
0421     .sw_fini = cz_ih_sw_fini,
0422     .hw_init = cz_ih_hw_init,
0423     .hw_fini = cz_ih_hw_fini,
0424     .suspend = cz_ih_suspend,
0425     .resume = cz_ih_resume,
0426     .is_idle = cz_ih_is_idle,
0427     .wait_for_idle = cz_ih_wait_for_idle,
0428     .soft_reset = cz_ih_soft_reset,
0429     .set_clockgating_state = cz_ih_set_clockgating_state,
0430     .set_powergating_state = cz_ih_set_powergating_state,
0431 };
0432
0433 static const struct amdgpu_ih_funcs cz_ih_funcs = {
0434     .get_wptr = cz_ih_get_wptr,
0435     .decode_iv = cz_ih_decode_iv,
0436     .set_rptr = cz_ih_set_rptr
0437 };
0438
0439 static void cz_ih_set_interrupt_funcs(struct amdgpu_device *adev)
0440 {
0441     adev->irq.ih_funcs = &cz_ih_funcs;
0442 }
0443
0444 const struct amdgpu_ip_block_version cz_ih_ip_block =
0445 {
0446     .type = AMD_IP_BLOCK_TYPE_IH,
0447     .major = 3,
0448     .minor = 0,
0449     .rev = 0,
0450     .funcs = &cz_ih_ip_funcs,
0451 };