msm/adreno/a5xx_preempt.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /* Copyright (c) 2017 The Linux Foundation. All rights reserved.
0003  */
0004
0005 #include "msm_gem.h"
0006 #include "a5xx_gpu.h"
0007
0008 /*
0009  * Try to transition the preemption state from old to new. Return
0010  * true on success or false if the original state wasn't 'old'
0011  */
0012 static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
0013         enum preempt_state old, enum preempt_state new)
0014 {
0015     enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
0016         old, new);
0017
0018     return (cur == old);
0019 }
0020
0021 /*
0022  * Force the preemption state to the specified state.  This is used in cases
0023  * where the current state is known and won't change
0024  */
0025 static inline void set_preempt_state(struct a5xx_gpu *gpu,
0026         enum preempt_state new)
0027 {
0028     /*
0029      * preempt_state may be read by other cores trying to trigger a
0030      * preemption or in the interrupt handler so barriers are needed
0031      * before...
0032      */
0033     smp_mb__before_atomic();
0034     atomic_set(&gpu->preempt_state, new);
0035     /* ... and after*/
0036     smp_mb__after_atomic();
0037 }
0038
0039 /* Write the most recent wptr for the given ring into the hardware */
0040 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
0041 {
0042     unsigned long flags;
0043     uint32_t wptr;
0044
0045     if (!ring)
0046         return;
0047
0048     spin_lock_irqsave(&ring->preempt_lock, flags);
0049     wptr = get_wptr(ring);
0050     spin_unlock_irqrestore(&ring->preempt_lock, flags);
0051
0052     gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
0053 }
0054
0055 /* Return the highest priority ringbuffer with something in it */
0056 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
0057 {
0058     unsigned long flags;
0059     int i;
0060
0061     for (i = 0; i < gpu->nr_rings; i++) {
0062         bool empty;
0063         struct msm_ringbuffer *ring = gpu->rb[i];
0064
0065         spin_lock_irqsave(&ring->preempt_lock, flags);
0066         empty = (get_wptr(ring) == ring->memptrs->rptr);
0067         spin_unlock_irqrestore(&ring->preempt_lock, flags);
0068
0069         if (!empty)
0070             return ring;
0071     }
0072
0073     return NULL;
0074 }
0075
0076 static void a5xx_preempt_timer(struct timer_list *t)
0077 {
0078     struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
0079     struct msm_gpu *gpu = &a5xx_gpu->base.base;
0080     struct drm_device *dev = gpu->dev;
0081
0082     if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
0083         return;
0084
0085     DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
0086     kthread_queue_work(gpu->worker, &gpu->recover_work);
0087 }
0088
0089 /* Try to trigger a preemption switch */
0090 void a5xx_preempt_trigger(struct msm_gpu *gpu)
0091 {
0092     struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
0093     struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
0094     unsigned long flags;
0095     struct msm_ringbuffer *ring;
0096
0097     if (gpu->nr_rings == 1)
0098         return;
0099
0100     /*
0101      * Try to start preemption by moving from NONE to START. If
0102      * unsuccessful, a preemption is already in flight
0103      */
0104     if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
0105         return;
0106
0107     /* Get the next ring to preempt to */
0108     ring = get_next_ring(gpu);
0109
0110     /*
0111      * If no ring is populated or the highest priority ring is the current
0112      * one do nothing except to update the wptr to the latest and greatest
0113      */
0114     if (!ring || (a5xx_gpu->cur_ring == ring)) {
0115         /*
0116          * Its possible that while a preemption request is in progress
0117          * from an irq context, a user context trying to submit might
0118          * fail to update the write pointer, because it determines
0119          * that the preempt state is not PREEMPT_NONE.
0120          *
0121          * Close the race by introducing an intermediate
0122          * state PREEMPT_ABORT to let the submit path
0123          * know that the ringbuffer is not going to change
0124          * and can safely update the write pointer.
0125          */
0126
0127         set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
0128         update_wptr(gpu, a5xx_gpu->cur_ring);
0129         set_preempt_state(a5xx_gpu, PREEMPT_NONE);
0130         return;
0131     }
0132
0133     /* Make sure the wptr doesn't update while we're in motion */
0134     spin_lock_irqsave(&ring->preempt_lock, flags);
0135     a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
0136     spin_unlock_irqrestore(&ring->preempt_lock, flags);
0137
0138     /* Set the address of the incoming preemption record */
0139     gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
0140         REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
0141         a5xx_gpu->preempt_iova[ring->id]);
0142
0143     a5xx_gpu->next_ring = ring;
0144
0145     /* Start a timer to catch a stuck preemption */
0146     mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
0147
0148     /* Set the preemption state to triggered */
0149     set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
0150
0151     /* Make sure everything is written before hitting the button */
0152     wmb();
0153
0154     /* And actually start the preemption */
0155     gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
0156 }
0157
0158 void a5xx_preempt_irq(struct msm_gpu *gpu)
0159 {
0160     uint32_t status;
0161     struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
0162     struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
0163     struct drm_device *dev = gpu->dev;
0164
0165     if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
0166         return;
0167
0168     /* Delete the preemption watchdog timer */
0169     del_timer(&a5xx_gpu->preempt_timer);
0170
0171     /*
0172      * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
0173      * firing the interrupt, but there is a non zero chance of a hardware
0174      * condition or a software race that could set it again before we have a
0175      * chance to finish. If that happens, log and go for recovery
0176      */
0177     status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
0178     if (unlikely(status)) {
0179         set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
0180         DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
0181             gpu->name);
0182         kthread_queue_work(gpu->worker, &gpu->recover_work);
0183         return;
0184     }
0185
0186     a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
0187     a5xx_gpu->next_ring = NULL;
0188
0189     update_wptr(gpu, a5xx_gpu->cur_ring);
0190
0191     set_preempt_state(a5xx_gpu, PREEMPT_NONE);
0192 }
0193
0194 void a5xx_preempt_hw_init(struct msm_gpu *gpu)
0195 {
0196     struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
0197     struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
0198     int i;
0199
0200     /* Always come up on rb 0 */
0201     a5xx_gpu->cur_ring = gpu->rb[0];
0202
0203     /* No preemption if we only have one ring */
0204     if (gpu->nr_rings == 1)
0205         return;
0206
0207     for (i = 0; i < gpu->nr_rings; i++) {
0208         a5xx_gpu->preempt[i]->wptr = 0;
0209         a5xx_gpu->preempt[i]->rptr = 0;
0210         a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
0211     }
0212
0213     /* Write a 0 to signal that we aren't switching pagetables */
0214     gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
0215         REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);
0216
0217     /* Reset the preemption state */
0218     set_preempt_state(a5xx_gpu, PREEMPT_NONE);
0219 }
0220
0221 static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
0222         struct msm_ringbuffer *ring)
0223 {
0224     struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
0225     struct msm_gpu *gpu = &adreno_gpu->base;
0226     struct a5xx_preempt_record *ptr;
0227     void *counters;
0228     struct drm_gem_object *bo = NULL, *counters_bo = NULL;
0229     u64 iova = 0, counters_iova = 0;
0230
0231     ptr = msm_gem_kernel_new(gpu->dev,
0232         A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
0233         MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova);
0234
0235     if (IS_ERR(ptr))
0236         return PTR_ERR(ptr);
0237
0238     /* The buffer to store counters needs to be unprivileged */
0239     counters = msm_gem_kernel_new(gpu->dev,
0240         A5XX_PREEMPT_COUNTER_SIZE,
0241         MSM_BO_WC, gpu->aspace, &counters_bo, &counters_iova);
0242     if (IS_ERR(counters)) {
0243         msm_gem_kernel_put(bo, gpu->aspace);
0244         return PTR_ERR(counters);
0245     }
0246
0247     msm_gem_object_set_name(bo, "preempt");
0248     msm_gem_object_set_name(counters_bo, "preempt_counters");
0249
0250     a5xx_gpu->preempt_bo[ring->id] = bo;
0251     a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo;
0252     a5xx_gpu->preempt_iova[ring->id] = iova;
0253     a5xx_gpu->preempt[ring->id] = ptr;
0254
0255     /* Set up the defaults on the preemption record */
0256
0257     ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
0258     ptr->info = 0;
0259     ptr->data = 0;
0260     ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE;
0261
0262     ptr->rptr_addr = shadowptr(a5xx_gpu, ring);
0263     ptr->counter = counters_iova;
0264
0265     return 0;
0266 }
0267
0268 void a5xx_preempt_fini(struct msm_gpu *gpu)
0269 {
0270     struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
0271     struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
0272     int i;
0273
0274     for (i = 0; i < gpu->nr_rings; i++) {
0275         msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace);
0276         msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], gpu->aspace);
0277     }
0278 }
0279
0280 void a5xx_preempt_init(struct msm_gpu *gpu)
0281 {
0282     struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
0283     struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
0284     int i;
0285
0286     /* No preemption if we only have one ring */
0287     if (gpu->nr_rings <= 1)
0288         return;
0289
0290     for (i = 0; i < gpu->nr_rings; i++) {
0291         if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
0292             /*
0293              * On any failure our adventure is over. Clean up and
0294              * set nr_rings to 1 to force preemption off
0295              */
0296             a5xx_preempt_fini(gpu);
0297             gpu->nr_rings = 1;
0298
0299             return;
0300         }
0301     }
0302
0303     timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
0304 }