amd/amdgpu/iceland_ih.c

0001 /*
0002  * Copyright 2014 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  */
0023
0024 #include <linux/pci.h>
0025
0026 #include "amdgpu.h"
0027 #include "amdgpu_ih.h"
0028 #include "vid.h"
0029
0030 #include "oss/oss_2_4_d.h"
0031 #include "oss/oss_2_4_sh_mask.h"
0032
0033 #include "bif/bif_5_1_d.h"
0034 #include "bif/bif_5_1_sh_mask.h"
0035
0036 /*
0037  * Interrupts
0038  * Starting with r6xx, interrupts are handled via a ring buffer.
0039  * Ring buffers are areas of GPU accessible memory that the GPU
0040  * writes interrupt vectors into and the host reads vectors out of.
0041  * There is a rptr (read pointer) that determines where the
0042  * host is currently reading, and a wptr (write pointer)
0043  * which determines where the GPU has written.  When the
0044  * pointers are equal, the ring is idle.  When the GPU
0045  * writes vectors to the ring buffer, it increments the
0046  * wptr.  When there is an interrupt, the host then starts
0047  * fetching commands and processing them until the pointers are
0048  * equal again at which point it updates the rptr.
0049  */
0050
0051 static void iceland_ih_set_interrupt_funcs(struct amdgpu_device *adev);
0052
0053 /**
0054  * iceland_ih_enable_interrupts - Enable the interrupt ring buffer
0055  *
0056  * @adev: amdgpu_device pointer
0057  *
0058  * Enable the interrupt ring buffer (VI).
0059  */
0060 static void iceland_ih_enable_interrupts(struct amdgpu_device *adev)
0061 {
0062     u32 ih_cntl = RREG32(mmIH_CNTL);
0063     u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
0064
0065     ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 1);
0066     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
0067     WREG32(mmIH_CNTL, ih_cntl);
0068     WREG32(mmIH_RB_CNTL, ih_rb_cntl);
0069     adev->irq.ih.enabled = true;
0070 }
0071
0072 /**
0073  * iceland_ih_disable_interrupts - Disable the interrupt ring buffer
0074  *
0075  * @adev: amdgpu_device pointer
0076  *
0077  * Disable the interrupt ring buffer (VI).
0078  */
0079 static void iceland_ih_disable_interrupts(struct amdgpu_device *adev)
0080 {
0081     u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
0082     u32 ih_cntl = RREG32(mmIH_CNTL);
0083
0084     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
0085     ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 0);
0086     WREG32(mmIH_RB_CNTL, ih_rb_cntl);
0087     WREG32(mmIH_CNTL, ih_cntl);
0088     /* set rptr, wptr to 0 */
0089     WREG32(mmIH_RB_RPTR, 0);
0090     WREG32(mmIH_RB_WPTR, 0);
0091     adev->irq.ih.enabled = false;
0092     adev->irq.ih.rptr = 0;
0093 }
0094
0095 /**
0096  * iceland_ih_irq_init - init and enable the interrupt ring
0097  *
0098  * @adev: amdgpu_device pointer
0099  *
0100  * Allocate a ring buffer for the interrupt controller,
0101  * enable the RLC, disable interrupts, enable the IH
0102  * ring buffer and enable it (VI).
0103  * Called at device load and reume.
0104  * Returns 0 for success, errors for failure.
0105  */
0106 static int iceland_ih_irq_init(struct amdgpu_device *adev)
0107 {
0108     struct amdgpu_ih_ring *ih = &adev->irq.ih;
0109     int rb_bufsz;
0110     u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
0111
0112     /* disable irqs */
0113     iceland_ih_disable_interrupts(adev);
0114
0115     /* setup interrupt control */
0116     WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
0117     interrupt_cntl = RREG32(mmINTERRUPT_CNTL);
0118     /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
0119      * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
0120      */
0121     interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0);
0122     /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
0123     interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0);
0124     WREG32(mmINTERRUPT_CNTL, interrupt_cntl);
0125
0126     /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
0127     WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
0128
0129     rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
0130     ih_rb_cntl = REG_SET_FIELD(0, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
0131     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
0132     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
0133
0134     /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
0135     ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
0136
0137     /* set the writeback address whether it's enabled or not */
0138     WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr));
0139     WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF);
0140
0141     WREG32(mmIH_RB_CNTL, ih_rb_cntl);
0142
0143     /* set rptr, wptr to 0 */
0144     WREG32(mmIH_RB_RPTR, 0);
0145     WREG32(mmIH_RB_WPTR, 0);
0146
0147     /* Default settings for IH_CNTL (disabled at first) */
0148     ih_cntl = RREG32(mmIH_CNTL);
0149     ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, MC_VMID, 0);
0150
0151     if (adev->irq.msi_enabled)
0152         ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, RPTR_REARM, 1);
0153     WREG32(mmIH_CNTL, ih_cntl);
0154
0155     pci_set_master(adev->pdev);
0156
0157     /* enable interrupts */
0158     iceland_ih_enable_interrupts(adev);
0159
0160     return 0;
0161 }
0162
0163 /**
0164  * iceland_ih_irq_disable - disable interrupts
0165  *
0166  * @adev: amdgpu_device pointer
0167  *
0168  * Disable interrupts on the hw (VI).
0169  */
0170 static void iceland_ih_irq_disable(struct amdgpu_device *adev)
0171 {
0172     iceland_ih_disable_interrupts(adev);
0173
0174     /* Wait and acknowledge irq */
0175     mdelay(1);
0176 }
0177
0178 /**
0179  * iceland_ih_get_wptr - get the IH ring buffer wptr
0180  *
0181  * @adev: amdgpu_device pointer
0182  * @ih: IH ring buffer to fetch wptr
0183  *
0184  * Get the IH ring buffer wptr from either the register
0185  * or the writeback memory buffer (VI).  Also check for
0186  * ring buffer overflow and deal with it.
0187  * Used by cz_irq_process(VI).
0188  * Returns the value of the wptr.
0189  */
0190 static u32 iceland_ih_get_wptr(struct amdgpu_device *adev,
0191                    struct amdgpu_ih_ring *ih)
0192 {
0193     u32 wptr, tmp;
0194
0195     wptr = le32_to_cpu(*ih->wptr_cpu);
0196
0197     if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
0198         goto out;
0199
0200     /* Double check that the overflow wasn't already cleared. */
0201     wptr = RREG32(mmIH_RB_WPTR);
0202
0203     if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
0204         goto out;
0205
0206     wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
0207     /* When a ring buffer overflow happen start parsing interrupt
0208      * from the last not overwritten vector (wptr + 16). Hopefully
0209      * this should allow us to catchup.
0210      */
0211     dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
0212         wptr, ih->rptr, (wptr + 16) & ih->ptr_mask);
0213     ih->rptr = (wptr + 16) & ih->ptr_mask;
0214     tmp = RREG32(mmIH_RB_CNTL);
0215     tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
0216     WREG32(mmIH_RB_CNTL, tmp);
0217
0218
0219 out:
0220     return (wptr & ih->ptr_mask);
0221 }
0222
0223 /**
0224  * iceland_ih_decode_iv - decode an interrupt vector
0225  *
0226  * @adev: amdgpu_device pointer
0227  * @ih: IH ring buffer to decode
0228  * @entry: IV entry to place decoded information into
0229  *
0230  * Decodes the interrupt vector at the current rptr
0231  * position and also advance the position.
0232  */
0233 static void iceland_ih_decode_iv(struct amdgpu_device *adev,
0234                  struct amdgpu_ih_ring *ih,
0235                  struct amdgpu_iv_entry *entry)
0236 {
0237     /* wptr/rptr are in bytes! */
0238     u32 ring_index = ih->rptr >> 2;
0239     uint32_t dw[4];
0240
0241     dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
0242     dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
0243     dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
0244     dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
0245
0246     entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
0247     entry->src_id = dw[0] & 0xff;
0248     entry->src_data[0] = dw[1] & 0xfffffff;
0249     entry->ring_id = dw[2] & 0xff;
0250     entry->vmid = (dw[2] >> 8) & 0xff;
0251     entry->pasid = (dw[2] >> 16) & 0xffff;
0252
0253     /* wptr/rptr are in bytes! */
0254     ih->rptr += 16;
0255 }
0256
0257 /**
0258  * iceland_ih_set_rptr - set the IH ring buffer rptr
0259  *
0260  * @adev: amdgpu_device pointer
0261  * @ih: IH ring buffer to set rptr
0262  *
0263  * Set the IH ring buffer rptr.
0264  */
0265 static void iceland_ih_set_rptr(struct amdgpu_device *adev,
0266                 struct amdgpu_ih_ring *ih)
0267 {
0268     WREG32(mmIH_RB_RPTR, ih->rptr);
0269 }
0270
0271 static int iceland_ih_early_init(void *handle)
0272 {
0273     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0274     int ret;
0275
0276     ret = amdgpu_irq_add_domain(adev);
0277     if (ret)
0278         return ret;
0279
0280     iceland_ih_set_interrupt_funcs(adev);
0281
0282     return 0;
0283 }
0284
0285 static int iceland_ih_sw_init(void *handle)
0286 {
0287     int r;
0288     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0289
0290     r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, false);
0291     if (r)
0292         return r;
0293
0294     r = amdgpu_irq_init(adev);
0295
0296     return r;
0297 }
0298
0299 static int iceland_ih_sw_fini(void *handle)
0300 {
0301     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0302
0303     amdgpu_irq_fini_sw(adev);
0304     amdgpu_irq_remove_domain(adev);
0305
0306     return 0;
0307 }
0308
0309 static int iceland_ih_hw_init(void *handle)
0310 {
0311     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0312
0313     return iceland_ih_irq_init(adev);
0314 }
0315
0316 static int iceland_ih_hw_fini(void *handle)
0317 {
0318     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0319
0320     iceland_ih_irq_disable(adev);
0321
0322     return 0;
0323 }
0324
0325 static int iceland_ih_suspend(void *handle)
0326 {
0327     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0328
0329     return iceland_ih_hw_fini(adev);
0330 }
0331
0332 static int iceland_ih_resume(void *handle)
0333 {
0334     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0335
0336     return iceland_ih_hw_init(adev);
0337 }
0338
0339 static bool iceland_ih_is_idle(void *handle)
0340 {
0341     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0342     u32 tmp = RREG32(mmSRBM_STATUS);
0343
0344     if (REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))
0345         return false;
0346
0347     return true;
0348 }
0349
0350 static int iceland_ih_wait_for_idle(void *handle)
0351 {
0352     unsigned i;
0353     u32 tmp;
0354     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0355
0356     for (i = 0; i < adev->usec_timeout; i++) {
0357         /* read MC_STATUS */
0358         tmp = RREG32(mmSRBM_STATUS);
0359         if (!REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))
0360             return 0;
0361         udelay(1);
0362     }
0363     return -ETIMEDOUT;
0364 }
0365
0366 static int iceland_ih_soft_reset(void *handle)
0367 {
0368     u32 srbm_soft_reset = 0;
0369     struct amdgpu_device *adev = (struct amdgpu_device *)handle;
0370     u32 tmp = RREG32(mmSRBM_STATUS);
0371
0372     if (tmp & SRBM_STATUS__IH_BUSY_MASK)
0373         srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
0374                         SOFT_RESET_IH, 1);
0375
0376     if (srbm_soft_reset) {
0377         tmp = RREG32(mmSRBM_SOFT_RESET);
0378         tmp |= srbm_soft_reset;
0379         dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
0380         WREG32(mmSRBM_SOFT_RESET, tmp);
0381         tmp = RREG32(mmSRBM_SOFT_RESET);
0382
0383         udelay(50);
0384
0385         tmp &= ~srbm_soft_reset;
0386         WREG32(mmSRBM_SOFT_RESET, tmp);
0387         tmp = RREG32(mmSRBM_SOFT_RESET);
0388
0389         /* Wait a little for things to settle down */
0390         udelay(50);
0391     }
0392
0393     return 0;
0394 }
0395
0396 static int iceland_ih_set_clockgating_state(void *handle,
0397                       enum amd_clockgating_state state)
0398 {
0399     return 0;
0400 }
0401
0402 static int iceland_ih_set_powergating_state(void *handle,
0403                       enum amd_powergating_state state)
0404 {
0405     return 0;
0406 }
0407
0408 static const struct amd_ip_funcs iceland_ih_ip_funcs = {
0409     .name = "iceland_ih",
0410     .early_init = iceland_ih_early_init,
0411     .late_init = NULL,
0412     .sw_init = iceland_ih_sw_init,
0413     .sw_fini = iceland_ih_sw_fini,
0414     .hw_init = iceland_ih_hw_init,
0415     .hw_fini = iceland_ih_hw_fini,
0416     .suspend = iceland_ih_suspend,
0417     .resume = iceland_ih_resume,
0418     .is_idle = iceland_ih_is_idle,
0419     .wait_for_idle = iceland_ih_wait_for_idle,
0420     .soft_reset = iceland_ih_soft_reset,
0421     .set_clockgating_state = iceland_ih_set_clockgating_state,
0422     .set_powergating_state = iceland_ih_set_powergating_state,
0423 };
0424
0425 static const struct amdgpu_ih_funcs iceland_ih_funcs = {
0426     .get_wptr = iceland_ih_get_wptr,
0427     .decode_iv = iceland_ih_decode_iv,
0428     .set_rptr = iceland_ih_set_rptr
0429 };
0430
0431 static void iceland_ih_set_interrupt_funcs(struct amdgpu_device *adev)
0432 {
0433     adev->irq.ih_funcs = &iceland_ih_funcs;
0434 }
0435
0436 const struct amdgpu_ip_block_version iceland_ih_ip_block =
0437 {
0438     .type = AMD_IP_BLOCK_TYPE_IH,
0439     .major = 2,
0440     .minor = 4,
0441     .rev = 0,
0442     .funcs = &iceland_ih_ip_funcs,
0443 };