drm/radeon/ni_dma.c

0001 /*
0002  * Copyright 2010 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  * Authors: Alex Deucher
0023  */
0024
0025 #include "radeon.h"
0026 #include "radeon_asic.h"
0027 #include "radeon_trace.h"
0028 #include "ni.h"
0029 #include "nid.h"
0030
0031 /*
0032  * DMA
0033  * Starting with R600, the GPU has an asynchronous
0034  * DMA engine.  The programming model is very similar
0035  * to the 3D engine (ring buffer, IBs, etc.), but the
0036  * DMA controller has it's own packet format that is
0037  * different form the PM4 format used by the 3D engine.
0038  * It supports copying data, writing embedded data,
0039  * solid fills, and a number of other things.  It also
0040  * has support for tiling/detiling of buffers.
0041  * Cayman and newer support two asynchronous DMA engines.
0042  */
0043
0044 /**
0045  * cayman_dma_get_rptr - get the current read pointer
0046  *
0047  * @rdev: radeon_device pointer
0048  * @ring: radeon ring pointer
0049  *
0050  * Get the current rptr from the hardware (cayman+).
0051  */
0052 uint32_t cayman_dma_get_rptr(struct radeon_device *rdev,
0053                  struct radeon_ring *ring)
0054 {
0055     u32 rptr, reg;
0056
0057     if (rdev->wb.enabled) {
0058         rptr = rdev->wb.wb[ring->rptr_offs/4];
0059     } else {
0060         if (ring->idx == R600_RING_TYPE_DMA_INDEX)
0061             reg = DMA_RB_RPTR + DMA0_REGISTER_OFFSET;
0062         else
0063             reg = DMA_RB_RPTR + DMA1_REGISTER_OFFSET;
0064
0065         rptr = RREG32(reg);
0066     }
0067
0068     return (rptr & 0x3fffc) >> 2;
0069 }
0070
0071 /**
0072  * cayman_dma_get_wptr - get the current write pointer
0073  *
0074  * @rdev: radeon_device pointer
0075  * @ring: radeon ring pointer
0076  *
0077  * Get the current wptr from the hardware (cayman+).
0078  */
0079 uint32_t cayman_dma_get_wptr(struct radeon_device *rdev,
0080                struct radeon_ring *ring)
0081 {
0082     u32 reg;
0083
0084     if (ring->idx == R600_RING_TYPE_DMA_INDEX)
0085         reg = DMA_RB_WPTR + DMA0_REGISTER_OFFSET;
0086     else
0087         reg = DMA_RB_WPTR + DMA1_REGISTER_OFFSET;
0088
0089     return (RREG32(reg) & 0x3fffc) >> 2;
0090 }
0091
0092 /**
0093  * cayman_dma_set_wptr - commit the write pointer
0094  *
0095  * @rdev: radeon_device pointer
0096  * @ring: radeon ring pointer
0097  *
0098  * Write the wptr back to the hardware (cayman+).
0099  */
0100 void cayman_dma_set_wptr(struct radeon_device *rdev,
0101              struct radeon_ring *ring)
0102 {
0103     u32 reg;
0104
0105     if (ring->idx == R600_RING_TYPE_DMA_INDEX)
0106         reg = DMA_RB_WPTR + DMA0_REGISTER_OFFSET;
0107     else
0108         reg = DMA_RB_WPTR + DMA1_REGISTER_OFFSET;
0109
0110     WREG32(reg, (ring->wptr << 2) & 0x3fffc);
0111 }
0112
0113 /**
0114  * cayman_dma_ring_ib_execute - Schedule an IB on the DMA engine
0115  *
0116  * @rdev: radeon_device pointer
0117  * @ib: IB object to schedule
0118  *
0119  * Schedule an IB in the DMA ring (cayman-SI).
0120  */
0121 void cayman_dma_ring_ib_execute(struct radeon_device *rdev,
0122                 struct radeon_ib *ib)
0123 {
0124     struct radeon_ring *ring = &rdev->ring[ib->ring];
0125     unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
0126
0127     if (rdev->wb.enabled) {
0128         u32 next_rptr = ring->wptr + 4;
0129         while ((next_rptr & 7) != 5)
0130             next_rptr++;
0131         next_rptr += 3;
0132         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
0133         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
0134         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
0135         radeon_ring_write(ring, next_rptr);
0136     }
0137
0138     /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
0139      * Pad as necessary with NOPs.
0140      */
0141     while ((ring->wptr & 7) != 5)
0142         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
0143     radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, vm_id, 0));
0144     radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
0145     radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
0146
0147 }
0148
0149 /**
0150  * cayman_dma_stop - stop the async dma engines
0151  *
0152  * @rdev: radeon_device pointer
0153  *
0154  * Stop the async dma engines (cayman-SI).
0155  */
0156 void cayman_dma_stop(struct radeon_device *rdev)
0157 {
0158     u32 rb_cntl;
0159
0160     if ((rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) ||
0161         (rdev->asic->copy.copy_ring_index == CAYMAN_RING_TYPE_DMA1_INDEX))
0162         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
0163
0164     /* dma0 */
0165     rb_cntl = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
0166     rb_cntl &= ~DMA_RB_ENABLE;
0167     WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, rb_cntl);
0168
0169     /* dma1 */
0170     rb_cntl = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
0171     rb_cntl &= ~DMA_RB_ENABLE;
0172     WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, rb_cntl);
0173
0174     rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
0175     rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;
0176 }
0177
0178 /**
0179  * cayman_dma_resume - setup and start the async dma engines
0180  *
0181  * @rdev: radeon_device pointer
0182  *
0183  * Set up the DMA ring buffers and enable them. (cayman-SI).
0184  * Returns 0 for success, error for failure.
0185  */
0186 int cayman_dma_resume(struct radeon_device *rdev)
0187 {
0188     struct radeon_ring *ring;
0189     u32 rb_cntl, dma_cntl, ib_cntl;
0190     u32 rb_bufsz;
0191     u32 reg_offset, wb_offset;
0192     int i, r;
0193
0194     for (i = 0; i < 2; i++) {
0195         if (i == 0) {
0196             ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
0197             reg_offset = DMA0_REGISTER_OFFSET;
0198             wb_offset = R600_WB_DMA_RPTR_OFFSET;
0199         } else {
0200             ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
0201             reg_offset = DMA1_REGISTER_OFFSET;
0202             wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
0203         }
0204
0205         WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
0206         WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
0207
0208         /* Set ring buffer size in dwords */
0209         rb_bufsz = order_base_2(ring->ring_size / 4);
0210         rb_cntl = rb_bufsz << 1;
0211 #ifdef __BIG_ENDIAN
0212         rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
0213 #endif
0214         WREG32(DMA_RB_CNTL + reg_offset, rb_cntl);
0215
0216         /* Initialize the ring buffer's read and write pointers */
0217         WREG32(DMA_RB_RPTR + reg_offset, 0);
0218         WREG32(DMA_RB_WPTR + reg_offset, 0);
0219
0220         /* set the wb address whether it's enabled or not */
0221         WREG32(DMA_RB_RPTR_ADDR_HI + reg_offset,
0222                upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFF);
0223         WREG32(DMA_RB_RPTR_ADDR_LO + reg_offset,
0224                ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
0225
0226         if (rdev->wb.enabled)
0227             rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
0228
0229         WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8);
0230
0231         /* enable DMA IBs */
0232         ib_cntl = DMA_IB_ENABLE | CMD_VMID_FORCE;
0233 #ifdef __BIG_ENDIAN
0234         ib_cntl |= DMA_IB_SWAP_ENABLE;
0235 #endif
0236         WREG32(DMA_IB_CNTL + reg_offset, ib_cntl);
0237
0238         dma_cntl = RREG32(DMA_CNTL + reg_offset);
0239         dma_cntl &= ~CTXEMPTY_INT_ENABLE;
0240         WREG32(DMA_CNTL + reg_offset, dma_cntl);
0241
0242         ring->wptr = 0;
0243         WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2);
0244
0245         WREG32(DMA_RB_CNTL + reg_offset, rb_cntl | DMA_RB_ENABLE);
0246
0247         ring->ready = true;
0248
0249         r = radeon_ring_test(rdev, ring->idx, ring);
0250         if (r) {
0251             ring->ready = false;
0252             return r;
0253         }
0254     }
0255
0256     if ((rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) ||
0257         (rdev->asic->copy.copy_ring_index == CAYMAN_RING_TYPE_DMA1_INDEX))
0258         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
0259
0260     return 0;
0261 }
0262
0263 /**
0264  * cayman_dma_fini - tear down the async dma engines
0265  *
0266  * @rdev: radeon_device pointer
0267  *
0268  * Stop the async dma engines and free the rings (cayman-SI).
0269  */
0270 void cayman_dma_fini(struct radeon_device *rdev)
0271 {
0272     cayman_dma_stop(rdev);
0273     radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
0274     radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
0275 }
0276
0277 /**
0278  * cayman_dma_is_lockup - Check if the DMA engine is locked up
0279  *
0280  * @rdev: radeon_device pointer
0281  * @ring: radeon_ring structure holding ring information
0282  *
0283  * Check if the async DMA engine is locked up.
0284  * Returns true if the engine appears to be locked up, false if not.
0285  */
0286 bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
0287 {
0288     u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
0289     u32 mask;
0290
0291     if (ring->idx == R600_RING_TYPE_DMA_INDEX)
0292         mask = RADEON_RESET_DMA;
0293     else
0294         mask = RADEON_RESET_DMA1;
0295
0296     if (!(reset_mask & mask)) {
0297         radeon_ring_lockup_update(rdev, ring);
0298         return false;
0299     }
0300     return radeon_ring_test_lockup(rdev, ring);
0301 }
0302
0303 /**
0304  * cayman_dma_vm_copy_pages - update PTEs by copying them from the GART
0305  *
0306  * @rdev: radeon_device pointer
0307  * @ib: indirect buffer to fill with commands
0308  * @pe: addr of the page entry
0309  * @src: src addr where to copy from
0310  * @count: number of page entries to update
0311  *
0312  * Update PTEs by copying them from the GART using the DMA (cayman/TN).
0313  */
0314 void cayman_dma_vm_copy_pages(struct radeon_device *rdev,
0315                   struct radeon_ib *ib,
0316                   uint64_t pe, uint64_t src,
0317                   unsigned count)
0318 {
0319     unsigned ndw;
0320
0321     while (count) {
0322         ndw = count * 2;
0323         if (ndw > 0xFFFFE)
0324             ndw = 0xFFFFE;
0325
0326         ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY,
0327                               0, 0, ndw);
0328         ib->ptr[ib->length_dw++] = lower_32_bits(pe);
0329         ib->ptr[ib->length_dw++] = lower_32_bits(src);
0330         ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
0331         ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff;
0332
0333         pe += ndw * 4;
0334         src += ndw * 4;
0335         count -= ndw / 2;
0336     }
0337 }
0338
0339 /**
0340  * cayman_dma_vm_write_pages - update PTEs by writing them manually
0341  *
0342  * @rdev: radeon_device pointer
0343  * @ib: indirect buffer to fill with commands
0344  * @pe: addr of the page entry
0345  * @addr: dst addr to write into pe
0346  * @count: number of page entries to update
0347  * @incr: increase next addr by incr bytes
0348  * @flags: hw access flags
0349  *
0350  * Update PTEs by writing them manually using the DMA (cayman/TN).
0351  */
0352 void cayman_dma_vm_write_pages(struct radeon_device *rdev,
0353                    struct radeon_ib *ib,
0354                    uint64_t pe,
0355                    uint64_t addr, unsigned count,
0356                    uint32_t incr, uint32_t flags)
0357 {
0358     uint64_t value;
0359     unsigned ndw;
0360
0361     while (count) {
0362         ndw = count * 2;
0363         if (ndw > 0xFFFFE)
0364             ndw = 0xFFFFE;
0365
0366         /* for non-physically contiguous pages (system) */
0367         ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE,
0368                               0, 0, ndw);
0369         ib->ptr[ib->length_dw++] = pe;
0370         ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
0371         for (; ndw > 0; ndw -= 2, --count, pe += 8) {
0372             if (flags & R600_PTE_SYSTEM) {
0373                 value = radeon_vm_map_gart(rdev, addr);
0374             } else if (flags & R600_PTE_VALID) {
0375                 value = addr;
0376             } else {
0377                 value = 0;
0378             }
0379             addr += incr;
0380             value |= flags;
0381             ib->ptr[ib->length_dw++] = value;
0382             ib->ptr[ib->length_dw++] = upper_32_bits(value);
0383         }
0384     }
0385 }
0386
0387 /**
0388  * cayman_dma_vm_set_pages - update the page tables using the DMA
0389  *
0390  * @rdev: radeon_device pointer
0391  * @ib: indirect buffer to fill with commands
0392  * @pe: addr of the page entry
0393  * @addr: dst addr to write into pe
0394  * @count: number of page entries to update
0395  * @incr: increase next addr by incr bytes
0396  * @flags: hw access flags
0397  *
0398  * Update the page tables using the DMA (cayman/TN).
0399  */
0400 void cayman_dma_vm_set_pages(struct radeon_device *rdev,
0401                  struct radeon_ib *ib,
0402                  uint64_t pe,
0403                  uint64_t addr, unsigned count,
0404                  uint32_t incr, uint32_t flags)
0405 {
0406     uint64_t value;
0407     unsigned ndw;
0408
0409     while (count) {
0410         ndw = count * 2;
0411         if (ndw > 0xFFFFE)
0412             ndw = 0xFFFFE;
0413
0414         if (flags & R600_PTE_VALID)
0415             value = addr;
0416         else
0417             value = 0;
0418
0419         /* for physically contiguous pages (vram) */
0420         ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
0421         ib->ptr[ib->length_dw++] = pe; /* dst addr */
0422         ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
0423         ib->ptr[ib->length_dw++] = flags; /* mask */
0424         ib->ptr[ib->length_dw++] = 0;
0425         ib->ptr[ib->length_dw++] = value; /* value */
0426         ib->ptr[ib->length_dw++] = upper_32_bits(value);
0427         ib->ptr[ib->length_dw++] = incr; /* increment size */
0428         ib->ptr[ib->length_dw++] = 0;
0429
0430         pe += ndw * 4;
0431         addr += (ndw / 2) * incr;
0432         count -= ndw / 2;
0433     }
0434 }
0435
0436 /**
0437  * cayman_dma_vm_pad_ib - pad the IB to the required number of dw
0438  *
0439  * @ib: indirect buffer to fill with padding
0440  *
0441  */
0442 void cayman_dma_vm_pad_ib(struct radeon_ib *ib)
0443 {
0444     while (ib->length_dw & 0x7)
0445         ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0);
0446 }
0447
0448 void cayman_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
0449              unsigned vm_id, uint64_t pd_addr)
0450 {
0451     radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
0452     radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2));
0453     radeon_ring_write(ring, pd_addr >> 12);
0454
0455     /* flush hdp cache */
0456     radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
0457     radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
0458     radeon_ring_write(ring, 1);
0459
0460     /* bits 0-7 are the VM contexts0-7 */
0461     radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
0462     radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
0463     radeon_ring_write(ring, 1 << vm_id);
0464
0465     /* wait for invalidate to complete */
0466     radeon_ring_write(ring, DMA_SRBM_READ_PACKET);
0467     radeon_ring_write(ring, (0xff << 20) | (VM_INVALIDATE_REQUEST >> 2));
0468     radeon_ring_write(ring, 0); /* mask */
0469     radeon_ring_write(ring, 0); /* value */
0470 }
0471