amd/amdgpu/cik_ih.c

0001 /*
0002  * Copyright 2012 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023
0024 #include <linux/pci.h>
0025
0026 #include "amdgpu.h"
0027 #include "amdgpu_ih.h"
0028 #include "cikd.h"
0029
0030 #include "bif/bif_4_1_d.h"
0031 #include "bif/bif_4_1_sh_mask.h"
0032
0033 #include "oss/oss_2_0_d.h"
0034 #include "oss/oss_2_0_sh_mask.h"
0035
0036 /*
0037  * Interrupts
0038  * Starting with r6xx, interrupts are handled via a ring buffer.
0039  * Ring buffers are areas of GPU accessible memory that the GPU
0040  * writes interrupt vectors into and the host reads vectors out of.
0041  * There is a rptr (read pointer) that determines where the
0042  * host is currently reading, and a wptr (write pointer)
0043  * which determines where the GPU has written.  When the
0044  * pointers are equal, the ring is idle.  When the GPU
0045  * writes vectors to the ring buffer, it increments the
0046  * wptr.  When there is an interrupt, the host then starts
0047  * fetching commands and processing them until the pointers are
0048  * equal again at which point it updates the rptr.
0049  */
0050
0051 static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev);
0052
0053 /**
0054  * cik_ih_enable_interrupts - Enable the interrupt ring buffer
0055  *
0056  * @adev: amdgpu_device pointer
0057  *
0058  * Enable the interrupt ring buffer (CIK).
0059  */
0060 static void cik_ih_enable_interrupts(struct amdgpu_device *adev)
0061 {
0062     u32 ih_cntl = RREG32(mmIH_CNTL);
0063     u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
0064
0065     ih_cntl |= IH_CNTL__ENABLE_INTR_MASK;
0066     ih_rb_cntl |= IH_RB_CNTL__RB_ENABLE_MASK;
0067     WREG32(mmIH_CNTL, ih_cntl);
0068     WREG32(mmIH_RB_CNTL, ih_rb_cntl);
0069     adev->irq.ih.enabled = true;
0070 }
0071
0072 /**
0073  * cik_ih_disable_interrupts - Disable the interrupt ring buffer
0074  *
0075  * @adev: amdgpu_device pointer
0076  *
0077  * Disable the interrupt ring buffer (CIK).
0078  */
0079 static void cik_ih_disable_interrupts(struct amdgpu_device *adev)
0080 {
0081     u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
0082     u32 ih_cntl = RREG32(mmIH_CNTL);
0083
0084     ih_rb_cntl &= ~IH_RB_CNTL__RB_ENABLE_MASK;
0085     ih_cntl &= ~IH_CNTL__ENABLE_INTR_MASK;
0086     WREG32(mmIH_RB_CNTL, ih_rb_cntl);
0087     WREG32(mmIH_CNTL, ih_cntl);
0088     /* set rptr, wptr to 0 */
0089     WREG32(mmIH_RB_RPTR, 0);
0090     WREG32(mmIH_RB_WPTR, 0);
0091     adev->irq.ih.enabled = false;
0092     adev->irq.ih.rptr = 0;
0093 }
0094
0095 /**
0096  * cik_ih_irq_init - init and enable the interrupt ring
0097  *
0098  * @adev: amdgpu_device pointer
0099  *
0100  * Allocate a ring buffer for the interrupt controller,
0101  * enable the RLC, disable interrupts, enable the IH
0102  * ring buffer and enable it (CIK).
0103  * Called at device load and reume.
0104  * Returns 0 for success, errors for failure.
0105  */
0106 static int cik_ih_irq_init(struct amdgpu_device *adev)
0107 {
0108     struct amdgpu_ih_ring *ih = &adev->irq.ih;
0109     int rb_bufsz;
0110     u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
0111
0112     /* disable irqs */
0113     cik_ih_disable_interrupts(adev);
0114
0115     /* setup interrupt control */
0116     WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
0117     interrupt_cntl = RREG32(mmINTERRUPT_CNTL);
0118     /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
0119      * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
0120      */
0121     interrupt_cntl &= ~INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK;
0122     /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
0123     interrupt_cntl &= ~INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK;
0124     WREG32(mmINTERRUPT_CNTL, interrupt_cntl);
0125
0126     WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
0127     rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
0128
0129     ih_rb_cntl = (IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK |
0130               IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK |
0131               (rb_bufsz << 1));
0132
0133     ih_rb_cntl |= IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK;
0134
0135     /* set the writeback address whether it's enabled or not */
0136     WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr));
0137     WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF);
0138
0139     WREG32(mmIH_RB_CNTL, ih_rb_cntl);
0140
0141     /* set rptr, wptr to 0 */
0142     WREG32(mmIH_RB_RPTR, 0);
0143     WREG32(mmIH_RB_WPTR, 0);
0144
0145     /* Default settings for IH_CNTL (disabled at first) */
0146     ih_cntl = (0x10 << IH_CNTL__MC_WRREQ_CREDIT__SHIFT) |
0147         (0x10 << IH_CNTL__MC_WR_CLEAN_CNT__SHIFT) |
0148         (0 << IH_CNTL__MC_VMID__SHIFT);
0149     /* IH_CNTL__RPTR_REARM_MASK only works if msi's are enabled */
0150     if (adev->irq.msi_enabled)
0151         ih_cntl |= IH_CNTL__RPTR_REARM_MASK;
0152     WREG32(mmIH_CNTL, ih_cntl);
0153
0154     pci_set_master(adev->pdev);
0155
0156     /* enable irqs */
0157     cik_ih_enable_interrupts(adev);
0158
0159     return 0;
0160 }
0161
0162 /**
0163  * cik_ih_irq_disable - disable interrupts
0164  *
0165  * @adev: amdgpu_device pointer
0166  *
0167  * Disable interrupts on the hw (CIK).
0168  */
0169 static void cik_ih_irq_disable(struct amdgpu_device *adev)
0170 {
0171     cik_ih_disable_interrupts(adev);
0172     /* Wait and acknowledge irq */
0173     mdelay(1);
0174 }
0175
0176 /**
0177  * cik_ih_get_wptr - get the IH ring buffer wptr
0178  *
0179  * @adev: amdgpu_device pointer
0180  * @ih: IH ring buffer to fetch wptr
0181  *
0182  * Get the IH ring buffer wptr from either the register
0183  * or the writeback memory buffer (CIK).  Also check for
0184  * ring buffer overflow and deal with it.
0185  * Used by cik_irq_process().
0186  * Returns the value of the wptr.
0187  */
0188 static u32 cik_ih_get_wptr(struct amdgpu_device *adev,
0189                struct amdgpu_ih_ring *ih)
0190 {
0191     u32 wptr, tmp;
0192
0193     wptr = le32_to_cpu(*ih->wptr_cpu);
0194
0195     if (wptr & IH_RB_WPTR__RB_OVERFLOW_MASK) {
0196         wptr &= ~IH_RB_WPTR__RB_OVERFLOW_MASK;
0197         /* When a ring buffer overflow happen start parsing interrupt
0198          * from the last not overwritten vector (wptr + 16). Hopefully
0199          * this should allow us to catchup.
0200          */
0201         dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
0202              wptr, ih->rptr, (wptr + 16) & ih->ptr_mask);
0203         ih->rptr = (wptr + 16) & ih->ptr_mask;
0204         tmp = RREG32(mmIH_RB_CNTL);
0205         tmp |= IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK;
0206         WREG32(mmIH_RB_CNTL, tmp);
0207     }
0208     return (wptr & ih->ptr_mask);
0209 }
0210
0211 /*        CIK IV Ring
0212  * Each IV ring entry is 128 bits:
0213  * [7:0]    - interrupt source id
0214  * [31:8]   - reserved
0215  * [59:32]  - interrupt source data
0216  * [63:60]  - reserved
0217  * [71:64]  - RINGID
0218  *            CP:
0219  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
0220  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
0221  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
0222  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
0223  *            PIPE_ID - ME0 0=3D
0224  *                    - ME1&2 compute dispatcher (4 pipes each)
0225  *            SDMA:
0226  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
0227  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
0228  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
0229  * [79:72]  - VMID
0230  * [95:80]  - PASID
0231  * [127:96] - reserved
0232  */
0233
0234  /**
0235  * cik_ih_decode_iv - decode an interrupt vector
0236  *
0237  * @adev: amdgpu_device pointer
0238  *
0239  * Decodes the interrupt vector at the current rptr
0240  * position and also advance the position.
0241  */
0242 static void cik_ih_decode_iv(struct amdgpu_device *adev,
0243                  struct amdgpu_ih_ring *ih,
0244                  struct amdgpu_iv_entry *entry)
0245 {
0246     /* wptr/rptr are in bytes! */
0247     u32 ring_index = ih->rptr >> 2;
0248     uint32_t dw[4];
0249
0250     dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
0251     dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
0252     dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
0253     dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
0254
0255     entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
0256     entry->src_id = dw[0] & 0xff;
0257     entry->src_data[0] = dw[1] & 0xfffffff;
0258     entry->ring_id = dw[2] & 0xff;
0259     entry->vmid = (dw[2] >> 8) & 0xff;
0260     entry->pasid = (dw[2] >> 16) & 0xffff;
0261
0262     /* wptr/rptr are in bytes! */
0263     ih->rptr += 16;
0264 }
0265
0266 /**
0267  * cik_ih_set_rptr - set the IH ring buffer rptr
0268  *
0269  * @adev: amdgpu_device pointer
0270  * @ih: IH ring buffer to set wptr
0271  *
0272  * Set the IH ring buffer rptr.
0273  */
0274 static void cik_ih_set_rptr(struct amdgpu_device *adev,
0275                 struct amdgpu_ih_ring *ih)
0276 {
0277     WREG32(mmIH_RB_RPTR, ih->rptr);
0278 }
0279
0280 static int cik_ih_early_init(void *handle)
0281 {
0282     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0283     int ret;
0284
0285     ret = amdgpu_irq_add_domain(adev);
0286     if (ret)
0287         return ret;
0288
0289     cik_ih_set_interrupt_funcs(adev);
0290
0291     return 0;
0292 }
0293
0294 static int cik_ih_sw_init(void *handle)
0295 {
0296     int r;
0297     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0298
0299     r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, false);
0300     if (r)
0301         return r;
0302
0303     r = amdgpu_irq_init(adev);
0304
0305     return r;
0306 }
0307
0308 static int cik_ih_sw_fini(void *handle)
0309 {
0310     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0311
0312     amdgpu_irq_fini_sw(adev);
0313     amdgpu_irq_remove_domain(adev);
0314
0315     return 0;
0316 }
0317
0318 static int cik_ih_hw_init(void *handle)
0319 {
0320     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0321
0322     return cik_ih_irq_init(adev);
0323 }
0324
0325 static int cik_ih_hw_fini(void *handle)
0326 {
0327     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0328
0329     cik_ih_irq_disable(adev);
0330
0331     return 0;
0332 }
0333
0334 static int cik_ih_suspend(void *handle)
0335 {
0336     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0337
0338     return cik_ih_hw_fini(adev);
0339 }
0340
0341 static int cik_ih_resume(void *handle)
0342 {
0343     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0344
0345     return cik_ih_hw_init(adev);
0346 }
0347
0348 static bool cik_ih_is_idle(void *handle)
0349 {
0350     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0351     u32 tmp = RREG32(mmSRBM_STATUS);
0352
0353     if (tmp & SRBM_STATUS__IH_BUSY_MASK)
0354         return false;
0355
0356     return true;
0357 }
0358
0359 static int cik_ih_wait_for_idle(void *handle)
0360 {
0361     unsigned i;
0362     u32 tmp;
0363     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0364
0365     for (i = 0; i < adev->usec_timeout; i++) {
0366         /* read MC_STATUS */
0367         tmp = RREG32(mmSRBM_STATUS) & SRBM_STATUS__IH_BUSY_MASK;
0368         if (!tmp)
0369             return 0;
0370         udelay(1);
0371     }
0372     return -ETIMEDOUT;
0373 }
0374
0375 static int cik_ih_soft_reset(void *handle)
0376 {
0377     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0378
0379     u32 srbm_soft_reset = 0;
0380     u32 tmp = RREG32(mmSRBM_STATUS);
0381
0382     if (tmp & SRBM_STATUS__IH_BUSY_MASK)
0383         srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK;
0384
0385     if (srbm_soft_reset) {
0386         tmp = RREG32(mmSRBM_SOFT_RESET);
0387         tmp |= srbm_soft_reset;
0388         dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
0389         WREG32(mmSRBM_SOFT_RESET, tmp);
0390         tmp = RREG32(mmSRBM_SOFT_RESET);
0391
0392         udelay(50);
0393
0394         tmp &= ~srbm_soft_reset;
0395         WREG32(mmSRBM_SOFT_RESET, tmp);
0396         tmp = RREG32(mmSRBM_SOFT_RESET);
0397
0398         /* Wait a little for things to settle down */
0399         udelay(50);
0400     }
0401
0402     return 0;
0403 }
0404
0405 static int cik_ih_set_clockgating_state(void *handle,
0406                       enum amd_clockgating_state state)
0407 {
0408     return 0;
0409 }
0410
0411 static int cik_ih_set_powergating_state(void *handle,
0412                       enum amd_powergating_state state)
0413 {
0414     return 0;
0415 }
0416
0417 static const struct amd_ip_funcs cik_ih_ip_funcs = {
0418     .name = "cik_ih",
0419     .early_init = cik_ih_early_init,
0420     .late_init = NULL,
0421     .sw_init = cik_ih_sw_init,
0422     .sw_fini = cik_ih_sw_fini,
0423     .hw_init = cik_ih_hw_init,
0424     .hw_fini = cik_ih_hw_fini,
0425     .suspend = cik_ih_suspend,
0426     .resume = cik_ih_resume,
0427     .is_idle = cik_ih_is_idle,
0428     .wait_for_idle = cik_ih_wait_for_idle,
0429     .soft_reset = cik_ih_soft_reset,
0430     .set_clockgating_state = cik_ih_set_clockgating_state,
0431     .set_powergating_state = cik_ih_set_powergating_state,
0432 };
0433
0434 static const struct amdgpu_ih_funcs cik_ih_funcs = {
0435     .get_wptr = cik_ih_get_wptr,
0436     .decode_iv = cik_ih_decode_iv,
0437     .set_rptr = cik_ih_set_rptr
0438 };
0439
0440 static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev)
0441 {
0442     adev->irq.ih_funcs = &cik_ih_funcs;
0443 }
0444
0445 const struct amdgpu_ip_block_version cik_ih_ip_block =
0446 {
0447     .type = AMD_IP_BLOCK_TYPE_IH,
0448     .major = 2,
0449     .minor = 0,
0450     .rev = 0,
0451     .funcs = &cik_ih_ip_funcs,
0452 };