Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright 2008 Advanced Micro Devices, Inc.
0003  * Copyright 2008 Red Hat Inc.
0004  * Copyright 2009 Jerome Glisse.
0005  *
0006  * Permission is hereby granted, free of charge, to any person obtaining a
0007  * copy of this software and associated documentation files (the "Software"),
0008  * to deal in the Software without restriction, including without limitation
0009  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0010  * and/or sell copies of the Software, and to permit persons to whom the
0011  * Software is furnished to do so, subject to the following conditions:
0012  *
0013  * The above copyright notice and this permission notice shall be included in
0014  * all copies or substantial portions of the Software.
0015  *
0016  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0017  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0018  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0019  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0020  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0021  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0022  * OTHER DEALINGS IN THE SOFTWARE.
0023  *
0024  * Authors: Dave Airlie
0025  *          Alex Deucher
0026  *          Jerome Glisse
0027  *          Christian König
0028  */
0029 
0030 #include <drm/drm_device.h>
0031 #include <drm/drm_file.h>
0032 
0033 #include "radeon.h"
0034 
0035 /*
0036  * Rings
0037  * Most engines on the GPU are fed via ring buffers.  Ring
0038  * buffers are areas of GPU accessible memory that the host
0039  * writes commands into and the GPU reads commands out of.
0040  * There is a rptr (read pointer) that determines where the
0041  * GPU is currently reading, and a wptr (write pointer)
0042  * which determines where the host has written.  When the
0043  * pointers are equal, the ring is idle.  When the host
0044  * writes commands to the ring buffer, it increments the
0045  * wptr.  The GPU then starts fetching commands and executes
0046  * them until the pointers are equal again.
0047  */
0048 static void radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
0049 
0050 /**
0051  * radeon_ring_supports_scratch_reg - check if the ring supports
0052  * writing to scratch registers
0053  *
0054  * @rdev: radeon_device pointer
0055  * @ring: radeon_ring structure holding ring information
0056  *
0057  * Check if a specific ring supports writing to scratch registers (all asics).
0058  * Returns true if the ring supports writing to scratch regs, false if not.
0059  */
0060 bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
0061                       struct radeon_ring *ring)
0062 {
0063     switch (ring->idx) {
0064     case RADEON_RING_TYPE_GFX_INDEX:
0065     case CAYMAN_RING_TYPE_CP1_INDEX:
0066     case CAYMAN_RING_TYPE_CP2_INDEX:
0067         return true;
0068     default:
0069         return false;
0070     }
0071 }
0072 
0073 /**
0074  * radeon_ring_free_size - update the free size
0075  *
0076  * @rdev: radeon_device pointer
0077  * @ring: radeon_ring structure holding ring information
0078  *
0079  * Update the free dw slots in the ring buffer (all asics).
0080  */
0081 void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
0082 {
0083     uint32_t rptr = radeon_ring_get_rptr(rdev, ring);
0084 
0085     /* This works because ring_size is a power of 2 */
0086     ring->ring_free_dw = rptr + (ring->ring_size / 4);
0087     ring->ring_free_dw -= ring->wptr;
0088     ring->ring_free_dw &= ring->ptr_mask;
0089     if (!ring->ring_free_dw) {
0090         /* this is an empty ring */
0091         ring->ring_free_dw = ring->ring_size / 4;
0092         /*  update lockup info to avoid false positive */
0093         radeon_ring_lockup_update(rdev, ring);
0094     }
0095 }
0096 
0097 /**
0098  * radeon_ring_alloc - allocate space on the ring buffer
0099  *
0100  * @rdev: radeon_device pointer
0101  * @ring: radeon_ring structure holding ring information
0102  * @ndw: number of dwords to allocate in the ring buffer
0103  *
0104  * Allocate @ndw dwords in the ring buffer (all asics).
0105  * Returns 0 on success, error on failure.
0106  */
0107 int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
0108 {
0109     int r;
0110 
0111     /* make sure we aren't trying to allocate more space than there is on the ring */
0112     if (ndw > (ring->ring_size / 4))
0113         return -ENOMEM;
0114     /* Align requested size with padding so unlock_commit can
0115      * pad safely */
0116     radeon_ring_free_size(rdev, ring);
0117     ndw = (ndw + ring->align_mask) & ~ring->align_mask;
0118     while (ndw > (ring->ring_free_dw - 1)) {
0119         radeon_ring_free_size(rdev, ring);
0120         if (ndw < ring->ring_free_dw) {
0121             break;
0122         }
0123         r = radeon_fence_wait_next(rdev, ring->idx);
0124         if (r)
0125             return r;
0126     }
0127     ring->count_dw = ndw;
0128     ring->wptr_old = ring->wptr;
0129     return 0;
0130 }
0131 
0132 /**
0133  * radeon_ring_lock - lock the ring and allocate space on it
0134  *
0135  * @rdev: radeon_device pointer
0136  * @ring: radeon_ring structure holding ring information
0137  * @ndw: number of dwords to allocate in the ring buffer
0138  *
0139  * Lock the ring and allocate @ndw dwords in the ring buffer
0140  * (all asics).
0141  * Returns 0 on success, error on failure.
0142  */
0143 int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
0144 {
0145     int r;
0146 
0147     mutex_lock(&rdev->ring_lock);
0148     r = radeon_ring_alloc(rdev, ring, ndw);
0149     if (r) {
0150         mutex_unlock(&rdev->ring_lock);
0151         return r;
0152     }
0153     return 0;
0154 }
0155 
0156 /**
0157  * radeon_ring_commit - tell the GPU to execute the new
0158  * commands on the ring buffer
0159  *
0160  * @rdev: radeon_device pointer
0161  * @ring: radeon_ring structure holding ring information
0162  * @hdp_flush: Whether or not to perform an HDP cache flush
0163  *
0164  * Update the wptr (write pointer) to tell the GPU to
0165  * execute new commands on the ring buffer (all asics).
0166  */
0167 void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring,
0168             bool hdp_flush)
0169 {
0170     /* If we are emitting the HDP flush via the ring buffer, we need to
0171      * do it before padding.
0172      */
0173     if (hdp_flush && rdev->asic->ring[ring->idx]->hdp_flush)
0174         rdev->asic->ring[ring->idx]->hdp_flush(rdev, ring);
0175     /* We pad to match fetch size */
0176     while (ring->wptr & ring->align_mask) {
0177         radeon_ring_write(ring, ring->nop);
0178     }
0179     mb();
0180     /* If we are emitting the HDP flush via MMIO, we need to do it after
0181      * all CPU writes to VRAM finished.
0182      */
0183     if (hdp_flush && rdev->asic->mmio_hdp_flush)
0184         rdev->asic->mmio_hdp_flush(rdev);
0185     radeon_ring_set_wptr(rdev, ring);
0186 }
0187 
0188 /**
0189  * radeon_ring_unlock_commit - tell the GPU to execute the new
0190  * commands on the ring buffer and unlock it
0191  *
0192  * @rdev: radeon_device pointer
0193  * @ring: radeon_ring structure holding ring information
0194  * @hdp_flush: Whether or not to perform an HDP cache flush
0195  *
0196  * Call radeon_ring_commit() then unlock the ring (all asics).
0197  */
0198 void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring,
0199                    bool hdp_flush)
0200 {
0201     radeon_ring_commit(rdev, ring, hdp_flush);
0202     mutex_unlock(&rdev->ring_lock);
0203 }
0204 
0205 /**
0206  * radeon_ring_undo - reset the wptr
0207  *
0208  * @ring: radeon_ring structure holding ring information
0209  *
0210  * Reset the driver's copy of the wptr (all asics).
0211  */
0212 void radeon_ring_undo(struct radeon_ring *ring)
0213 {
0214     ring->wptr = ring->wptr_old;
0215 }
0216 
0217 /**
0218  * radeon_ring_unlock_undo - reset the wptr and unlock the ring
0219  *
0220  * @rdev:       radeon device structure
0221  * @ring: radeon_ring structure holding ring information
0222  *
0223  * Call radeon_ring_undo() then unlock the ring (all asics).
0224  */
0225 void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)
0226 {
0227     radeon_ring_undo(ring);
0228     mutex_unlock(&rdev->ring_lock);
0229 }
0230 
0231 /**
0232  * radeon_ring_lockup_update - update lockup variables
0233  *
0234  * @rdev:       radeon device structure
0235  * @ring: radeon_ring structure holding ring information
0236  *
0237  * Update the last rptr value and timestamp (all asics).
0238  */
0239 void radeon_ring_lockup_update(struct radeon_device *rdev,
0240                    struct radeon_ring *ring)
0241 {
0242     atomic_set(&ring->last_rptr, radeon_ring_get_rptr(rdev, ring));
0243     atomic64_set(&ring->last_activity, jiffies_64);
0244 }
0245 
0246 /**
0247  * radeon_ring_test_lockup() - check if ring is lockedup by recording information
0248  * @rdev:       radeon device structure
0249  * @ring:       radeon_ring structure holding ring information
0250  *
0251  */
0252 bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
0253 {
0254     uint32_t rptr = radeon_ring_get_rptr(rdev, ring);
0255     uint64_t last = atomic64_read(&ring->last_activity);
0256     uint64_t elapsed;
0257 
0258     if (rptr != atomic_read(&ring->last_rptr)) {
0259         /* ring is still working, no lockup */
0260         radeon_ring_lockup_update(rdev, ring);
0261         return false;
0262     }
0263 
0264     elapsed = jiffies_to_msecs(jiffies_64 - last);
0265     if (radeon_lockup_timeout && elapsed >= radeon_lockup_timeout) {
0266         dev_err(rdev->dev, "ring %d stalled for more than %llumsec\n",
0267             ring->idx, elapsed);
0268         return true;
0269     }
0270     /* give a chance to the GPU ... */
0271     return false;
0272 }
0273 
0274 /**
0275  * radeon_ring_backup - Back up the content of a ring
0276  *
0277  * @rdev: radeon_device pointer
0278  * @ring: the ring we want to back up
0279  * @data: placeholder for returned commit data
0280  *
0281  * Saves all unprocessed commits from a ring, returns the number of dwords saved.
0282  */
0283 unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
0284                 uint32_t **data)
0285 {
0286     unsigned size, ptr, i;
0287 
0288     /* just in case lock the ring */
0289     mutex_lock(&rdev->ring_lock);
0290     *data = NULL;
0291 
0292     if (ring->ring_obj == NULL) {
0293         mutex_unlock(&rdev->ring_lock);
0294         return 0;
0295     }
0296 
0297     /* it doesn't make sense to save anything if all fences are signaled */
0298     if (!radeon_fence_count_emitted(rdev, ring->idx)) {
0299         mutex_unlock(&rdev->ring_lock);
0300         return 0;
0301     }
0302 
0303     /* calculate the number of dw on the ring */
0304     if (ring->rptr_save_reg)
0305         ptr = RREG32(ring->rptr_save_reg);
0306     else if (rdev->wb.enabled)
0307         ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);
0308     else {
0309         /* no way to read back the next rptr */
0310         mutex_unlock(&rdev->ring_lock);
0311         return 0;
0312     }
0313 
0314     size = ring->wptr + (ring->ring_size / 4);
0315     size -= ptr;
0316     size &= ring->ptr_mask;
0317     if (size == 0) {
0318         mutex_unlock(&rdev->ring_lock);
0319         return 0;
0320     }
0321 
0322     /* and then save the content of the ring */
0323     *data = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
0324     if (!*data) {
0325         mutex_unlock(&rdev->ring_lock);
0326         return 0;
0327     }
0328     for (i = 0; i < size; ++i) {
0329         (*data)[i] = ring->ring[ptr++];
0330         ptr &= ring->ptr_mask;
0331     }
0332 
0333     mutex_unlock(&rdev->ring_lock);
0334     return size;
0335 }
0336 
0337 /**
0338  * radeon_ring_restore - append saved commands to the ring again
0339  *
0340  * @rdev: radeon_device pointer
0341  * @ring: ring to append commands to
0342  * @size: number of dwords we want to write
0343  * @data: saved commands
0344  *
0345  * Allocates space on the ring and restore the previously saved commands.
0346  */
0347 int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
0348             unsigned size, uint32_t *data)
0349 {
0350     int i, r;
0351 
0352     if (!size || !data)
0353         return 0;
0354 
0355     /* restore the saved ring content */
0356     r = radeon_ring_lock(rdev, ring, size);
0357     if (r)
0358         return r;
0359 
0360     for (i = 0; i < size; ++i) {
0361         radeon_ring_write(ring, data[i]);
0362     }
0363 
0364     radeon_ring_unlock_commit(rdev, ring, false);
0365     kvfree(data);
0366     return 0;
0367 }
0368 
0369 /**
0370  * radeon_ring_init - init driver ring struct.
0371  *
0372  * @rdev: radeon_device pointer
0373  * @ring: radeon_ring structure holding ring information
0374  * @ring_size: size of the ring
0375  * @rptr_offs: offset of the rptr writeback location in the WB buffer
0376  * @nop: nop packet for this ring
0377  *
0378  * Initialize the driver information for the selected ring (all asics).
0379  * Returns 0 on success, error on failure.
0380  */
0381 int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,
0382              unsigned rptr_offs, u32 nop)
0383 {
0384     int r;
0385 
0386     ring->ring_size = ring_size;
0387     ring->rptr_offs = rptr_offs;
0388     ring->nop = nop;
0389     ring->rdev = rdev;
0390     /* Allocate ring buffer */
0391     if (ring->ring_obj == NULL) {
0392         r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,
0393                      RADEON_GEM_DOMAIN_GTT, 0, NULL,
0394                      NULL, &ring->ring_obj);
0395         if (r) {
0396             dev_err(rdev->dev, "(%d) ring create failed\n", r);
0397             return r;
0398         }
0399         r = radeon_bo_reserve(ring->ring_obj, false);
0400         if (unlikely(r != 0))
0401             return r;
0402         r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,
0403                     &ring->gpu_addr);
0404         if (r) {
0405             radeon_bo_unreserve(ring->ring_obj);
0406             dev_err(rdev->dev, "(%d) ring pin failed\n", r);
0407             return r;
0408         }
0409         r = radeon_bo_kmap(ring->ring_obj,
0410                        (void **)&ring->ring);
0411         radeon_bo_unreserve(ring->ring_obj);
0412         if (r) {
0413             dev_err(rdev->dev, "(%d) ring map failed\n", r);
0414             return r;
0415         }
0416     }
0417     ring->ptr_mask = (ring->ring_size / 4) - 1;
0418     ring->ring_free_dw = ring->ring_size / 4;
0419     if (rdev->wb.enabled) {
0420         u32 index = RADEON_WB_RING0_NEXT_RPTR + (ring->idx * 4);
0421         ring->next_rptr_gpu_addr = rdev->wb.gpu_addr + index;
0422         ring->next_rptr_cpu_addr = &rdev->wb.wb[index/4];
0423     }
0424     radeon_debugfs_ring_init(rdev, ring);
0425     radeon_ring_lockup_update(rdev, ring);
0426     return 0;
0427 }
0428 
0429 /**
0430  * radeon_ring_fini - tear down the driver ring struct.
0431  *
0432  * @rdev: radeon_device pointer
0433  * @ring: radeon_ring structure holding ring information
0434  *
0435  * Tear down the driver information for the selected ring (all asics).
0436  */
0437 void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)
0438 {
0439     int r;
0440     struct radeon_bo *ring_obj;
0441 
0442     mutex_lock(&rdev->ring_lock);
0443     ring_obj = ring->ring_obj;
0444     ring->ready = false;
0445     ring->ring = NULL;
0446     ring->ring_obj = NULL;
0447     mutex_unlock(&rdev->ring_lock);
0448 
0449     if (ring_obj) {
0450         r = radeon_bo_reserve(ring_obj, false);
0451         if (likely(r == 0)) {
0452             radeon_bo_kunmap(ring_obj);
0453             radeon_bo_unpin(ring_obj);
0454             radeon_bo_unreserve(ring_obj);
0455         }
0456         radeon_bo_unref(&ring_obj);
0457     }
0458 }
0459 
0460 /*
0461  * Debugfs info
0462  */
0463 #if defined(CONFIG_DEBUG_FS)
0464 
0465 static int radeon_debugfs_ring_info_show(struct seq_file *m, void *unused)
0466 {
0467     struct radeon_ring *ring = (struct radeon_ring *) m->private;
0468     struct radeon_device *rdev = ring->rdev;
0469 
0470     uint32_t rptr, wptr, rptr_next;
0471     unsigned count, i, j;
0472 
0473     radeon_ring_free_size(rdev, ring);
0474     count = (ring->ring_size / 4) - ring->ring_free_dw;
0475 
0476     wptr = radeon_ring_get_wptr(rdev, ring);
0477     seq_printf(m, "wptr: 0x%08x [%5d]\n",
0478            wptr, wptr);
0479 
0480     rptr = radeon_ring_get_rptr(rdev, ring);
0481     seq_printf(m, "rptr: 0x%08x [%5d]\n",
0482            rptr, rptr);
0483 
0484     if (ring->rptr_save_reg) {
0485         rptr_next = RREG32(ring->rptr_save_reg);
0486         seq_printf(m, "rptr next(0x%04x): 0x%08x [%5d]\n",
0487                ring->rptr_save_reg, rptr_next, rptr_next);
0488     } else
0489         rptr_next = ~0;
0490 
0491     seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n",
0492            ring->wptr, ring->wptr);
0493     seq_printf(m, "last semaphore signal addr : 0x%016llx\n",
0494            ring->last_semaphore_signal_addr);
0495     seq_printf(m, "last semaphore wait addr   : 0x%016llx\n",
0496            ring->last_semaphore_wait_addr);
0497     seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
0498     seq_printf(m, "%u dwords in ring\n", count);
0499 
0500     if (!ring->ring)
0501         return 0;
0502 
0503     /* print 8 dw before current rptr as often it's the last executed
0504      * packet that is the root issue
0505      */
0506     i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;
0507     for (j = 0; j <= (count + 32); j++) {
0508         seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);
0509         if (rptr == i)
0510             seq_puts(m, " *");
0511         if (rptr_next == i)
0512             seq_puts(m, " #");
0513         seq_puts(m, "\n");
0514         i = (i + 1) & ring->ptr_mask;
0515     }
0516     return 0;
0517 }
0518 
0519 DEFINE_SHOW_ATTRIBUTE(radeon_debugfs_ring_info);
0520 
0521 static const char *radeon_debugfs_ring_idx_to_name(uint32_t ridx)
0522 {
0523     switch (ridx) {
0524     case RADEON_RING_TYPE_GFX_INDEX:
0525         return "radeon_ring_gfx";
0526     case CAYMAN_RING_TYPE_CP1_INDEX:
0527         return "radeon_ring_cp1";
0528     case CAYMAN_RING_TYPE_CP2_INDEX:
0529         return "radeon_ring_cp2";
0530     case R600_RING_TYPE_DMA_INDEX:
0531         return "radeon_ring_dma1";
0532     case CAYMAN_RING_TYPE_DMA1_INDEX:
0533         return "radeon_ring_dma2";
0534     case R600_RING_TYPE_UVD_INDEX:
0535         return "radeon_ring_uvd";
0536     case TN_RING_TYPE_VCE1_INDEX:
0537         return "radeon_ring_vce1";
0538     case TN_RING_TYPE_VCE2_INDEX:
0539         return "radeon_ring_vce2";
0540     default:
0541         return NULL;
0542 
0543     }
0544 }
0545 #endif
0546 
0547 static void radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)
0548 {
0549 #if defined(CONFIG_DEBUG_FS)
0550     const char *ring_name = radeon_debugfs_ring_idx_to_name(ring->idx);
0551     struct dentry *root = rdev->ddev->primary->debugfs_root;
0552 
0553     if (ring_name)
0554         debugfs_create_file(ring_name, 0444, root, ring,
0555                     &radeon_debugfs_ring_info_fops);
0556 
0557 #endif
0558 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!