drm/i915/i915_sw_fence.c

0001 /*
0002  * SPDX-License-Identifier: MIT
0003  *
0004  * (C) Copyright 2016 Intel Corporation
0005  */
0006
0007 #include <linux/slab.h>
0008 #include <linux/dma-fence.h>
0009 #include <linux/irq_work.h>
0010 #include <linux/dma-resv.h>
0011
0012 #include "i915_sw_fence.h"
0013 #include "i915_selftest.h"
0014
0015 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
0016 #define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr)
0017 #else
0018 #define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
0019 #endif
0020
0021 #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
0022 static DEFINE_SPINLOCK(i915_sw_fence_lock);
0023 #endif
0024
0025 #define WQ_FLAG_BITS \
0026     BITS_PER_TYPE(typeof_member(struct wait_queue_entry, flags))
0027
0028 /* after WQ_FLAG_* for safety */
0029 #define I915_SW_FENCE_FLAG_FENCE BIT(WQ_FLAG_BITS - 1)
0030 #define I915_SW_FENCE_FLAG_ALLOC BIT(WQ_FLAG_BITS - 2)
0031
0032 enum {
0033     DEBUG_FENCE_IDLE = 0,
0034     DEBUG_FENCE_NOTIFY,
0035 };
0036
0037 static void *i915_sw_fence_debug_hint(void *addr)
0038 {
0039     return (void *)(((struct i915_sw_fence *)addr)->fn);
0040 }
0041
0042 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
0043
0044 static const struct debug_obj_descr i915_sw_fence_debug_descr = {
0045     .name = "i915_sw_fence",
0046     .debug_hint = i915_sw_fence_debug_hint,
0047 };
0048
0049 static inline void debug_fence_init(struct i915_sw_fence *fence)
0050 {
0051     debug_object_init(fence, &i915_sw_fence_debug_descr);
0052 }
0053
0054 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
0055 {
0056     debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
0057 }
0058
0059 static inline void debug_fence_activate(struct i915_sw_fence *fence)
0060 {
0061     debug_object_activate(fence, &i915_sw_fence_debug_descr);
0062 }
0063
0064 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
0065                      int old, int new)
0066 {
0067     debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
0068 }
0069
0070 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
0071 {
0072     debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
0073 }
0074
0075 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
0076 {
0077     debug_object_destroy(fence, &i915_sw_fence_debug_descr);
0078 }
0079
0080 static inline void debug_fence_free(struct i915_sw_fence *fence)
0081 {
0082     debug_object_free(fence, &i915_sw_fence_debug_descr);
0083     smp_wmb(); /* flush the change in state before reallocation */
0084 }
0085
0086 static inline void debug_fence_assert(struct i915_sw_fence *fence)
0087 {
0088     debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
0089 }
0090
0091 #else
0092
0093 static inline void debug_fence_init(struct i915_sw_fence *fence)
0094 {
0095 }
0096
0097 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
0098 {
0099 }
0100
0101 static inline void debug_fence_activate(struct i915_sw_fence *fence)
0102 {
0103 }
0104
0105 static inline void debug_fence_set_state(struct i915_sw_fence *fence,
0106                      int old, int new)
0107 {
0108 }
0109
0110 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
0111 {
0112 }
0113
0114 static inline void debug_fence_destroy(struct i915_sw_fence *fence)
0115 {
0116 }
0117
0118 static inline void debug_fence_free(struct i915_sw_fence *fence)
0119 {
0120 }
0121
0122 static inline void debug_fence_assert(struct i915_sw_fence *fence)
0123 {
0124 }
0125
0126 #endif
0127
0128 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
0129                   enum i915_sw_fence_notify state)
0130 {
0131     return fence->fn(fence, state);
0132 }
0133
0134 #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS
0135 void i915_sw_fence_fini(struct i915_sw_fence *fence)
0136 {
0137     debug_fence_free(fence);
0138 }
0139 #endif
0140
0141 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
0142                     struct list_head *continuation)
0143 {
0144     wait_queue_head_t *x = &fence->wait;
0145     wait_queue_entry_t *pos, *next;
0146     unsigned long flags;
0147
0148     debug_fence_deactivate(fence);
0149     atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */
0150
0151     /*
0152      * To prevent unbounded recursion as we traverse the graph of
0153      * i915_sw_fences, we move the entry list from this, the next ready
0154      * fence, to the tail of the original fence's entry list
0155      * (and so added to the list to be woken).
0156      */
0157
0158     spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
0159     if (continuation) {
0160         list_for_each_entry_safe(pos, next, &x->head, entry) {
0161             if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
0162                 list_move_tail(&pos->entry, continuation);
0163             else
0164                 pos->func(pos, TASK_NORMAL, 0, continuation);
0165         }
0166     } else {
0167         LIST_HEAD(extra);
0168
0169         do {
0170             list_for_each_entry_safe(pos, next, &x->head, entry) {
0171                 int wake_flags;
0172
0173                 wake_flags = 0;
0174                 if (pos->flags & I915_SW_FENCE_FLAG_FENCE)
0175                     wake_flags = fence->error;
0176
0177                 pos->func(pos, TASK_NORMAL, wake_flags, &extra);
0178             }
0179
0180             if (list_empty(&extra))
0181                 break;
0182
0183             list_splice_tail_init(&extra, &x->head);
0184         } while (1);
0185     }
0186     spin_unlock_irqrestore(&x->lock, flags);
0187
0188     debug_fence_assert(fence);
0189 }
0190
0191 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
0192                      struct list_head *continuation)
0193 {
0194     debug_fence_assert(fence);
0195
0196     if (!atomic_dec_and_test(&fence->pending))
0197         return;
0198
0199     debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);
0200
0201     if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
0202         return;
0203
0204     debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);
0205
0206     __i915_sw_fence_wake_up_all(fence, continuation);
0207
0208     debug_fence_destroy(fence);
0209     __i915_sw_fence_notify(fence, FENCE_FREE);
0210 }
0211
0212 void i915_sw_fence_complete(struct i915_sw_fence *fence)
0213 {
0214     debug_fence_assert(fence);
0215
0216     if (WARN_ON(i915_sw_fence_done(fence)))
0217         return;
0218
0219     __i915_sw_fence_complete(fence, NULL);
0220 }
0221
0222 bool i915_sw_fence_await(struct i915_sw_fence *fence)
0223 {
0224     int pending;
0225
0226     /*
0227      * It is only safe to add a new await to the fence while it has
0228      * not yet been signaled (i.e. there are still existing signalers).
0229      */
0230     pending = atomic_read(&fence->pending);
0231     do {
0232         if (pending < 1)
0233             return false;
0234     } while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1));
0235
0236     return true;
0237 }
0238
0239 void __i915_sw_fence_init(struct i915_sw_fence *fence,
0240               i915_sw_fence_notify_t fn,
0241               const char *name,
0242               struct lock_class_key *key)
0243 {
0244     BUG_ON(!fn);
0245
0246     __init_waitqueue_head(&fence->wait, name, key);
0247     fence->fn = fn;
0248 #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
0249     fence->flags = 0;
0250 #endif
0251
0252     i915_sw_fence_reinit(fence);
0253 }
0254
0255 void i915_sw_fence_reinit(struct i915_sw_fence *fence)
0256 {
0257     debug_fence_init(fence);
0258
0259     atomic_set(&fence->pending, 1);
0260     fence->error = 0;
0261
0262     I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head));
0263 }
0264
0265 void i915_sw_fence_commit(struct i915_sw_fence *fence)
0266 {
0267     debug_fence_activate(fence);
0268     i915_sw_fence_complete(fence);
0269 }
0270
0271 static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
0272 {
0273     i915_sw_fence_set_error_once(wq->private, flags);
0274
0275     list_del(&wq->entry);
0276     __i915_sw_fence_complete(wq->private, key);
0277
0278     if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
0279         kfree(wq);
0280     return 0;
0281 }
0282
0283 #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
0284 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
0285                     const struct i915_sw_fence * const signaler)
0286 {
0287     wait_queue_entry_t *wq;
0288
0289     if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
0290         return false;
0291
0292     if (fence == signaler)
0293         return true;
0294
0295     list_for_each_entry(wq, &fence->wait.head, entry) {
0296         if (wq->func != i915_sw_fence_wake)
0297             continue;
0298
0299         if (__i915_sw_fence_check_if_after(wq->private, signaler))
0300             return true;
0301     }
0302
0303     return false;
0304 }
0305
0306 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
0307 {
0308     wait_queue_entry_t *wq;
0309
0310     if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
0311         return;
0312
0313     list_for_each_entry(wq, &fence->wait.head, entry) {
0314         if (wq->func != i915_sw_fence_wake)
0315             continue;
0316
0317         __i915_sw_fence_clear_checked_bit(wq->private);
0318     }
0319 }
0320
0321 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
0322                   const struct i915_sw_fence * const signaler)
0323 {
0324     unsigned long flags;
0325     bool err;
0326
0327     spin_lock_irqsave(&i915_sw_fence_lock, flags);
0328     err = __i915_sw_fence_check_if_after(fence, signaler);
0329     __i915_sw_fence_clear_checked_bit(fence);
0330     spin_unlock_irqrestore(&i915_sw_fence_lock, flags);
0331
0332     return err;
0333 }
0334 #else
0335 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
0336                      const struct i915_sw_fence * const signaler)
0337 {
0338     return false;
0339 }
0340 #endif
0341
0342 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
0343                       struct i915_sw_fence *signaler,
0344                       wait_queue_entry_t *wq, gfp_t gfp)
0345 {
0346     unsigned int pending;
0347     unsigned long flags;
0348
0349     debug_fence_assert(fence);
0350     might_sleep_if(gfpflags_allow_blocking(gfp));
0351
0352     if (i915_sw_fence_done(signaler)) {
0353         i915_sw_fence_set_error_once(fence, signaler->error);
0354         return 0;
0355     }
0356
0357     debug_fence_assert(signaler);
0358
0359     /* The dependency graph must be acyclic. */
0360     if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
0361         return -EINVAL;
0362
0363     pending = I915_SW_FENCE_FLAG_FENCE;
0364     if (!wq) {
0365         wq = kmalloc(sizeof(*wq), gfp);
0366         if (!wq) {
0367             if (!gfpflags_allow_blocking(gfp))
0368                 return -ENOMEM;
0369
0370             i915_sw_fence_wait(signaler);
0371             i915_sw_fence_set_error_once(fence, signaler->error);
0372             return 0;
0373         }
0374
0375         pending |= I915_SW_FENCE_FLAG_ALLOC;
0376     }
0377
0378     INIT_LIST_HEAD(&wq->entry);
0379     wq->flags = pending;
0380     wq->func = i915_sw_fence_wake;
0381     wq->private = fence;
0382
0383     i915_sw_fence_await(fence);
0384
0385     spin_lock_irqsave(&signaler->wait.lock, flags);
0386     if (likely(!i915_sw_fence_done(signaler))) {
0387         __add_wait_queue_entry_tail(&signaler->wait, wq);
0388         pending = 1;
0389     } else {
0390         i915_sw_fence_wake(wq, 0, signaler->error, NULL);
0391         pending = 0;
0392     }
0393     spin_unlock_irqrestore(&signaler->wait.lock, flags);
0394
0395     return pending;
0396 }
0397
0398 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
0399                  struct i915_sw_fence *signaler,
0400                  wait_queue_entry_t *wq)
0401 {
0402     return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
0403 }
0404
0405 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
0406                      struct i915_sw_fence *signaler,
0407                      gfp_t gfp)
0408 {
0409     return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
0410 }
0411
0412 struct i915_sw_dma_fence_cb_timer {
0413     struct i915_sw_dma_fence_cb base;
0414     struct dma_fence *dma;
0415     struct timer_list timer;
0416     struct irq_work work;
0417     struct rcu_head rcu;
0418 };
0419
0420 static void dma_i915_sw_fence_wake(struct dma_fence *dma,
0421                    struct dma_fence_cb *data)
0422 {
0423     struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
0424
0425     i915_sw_fence_set_error_once(cb->fence, dma->error);
0426     i915_sw_fence_complete(cb->fence);
0427     kfree(cb);
0428 }
0429
0430 static void timer_i915_sw_fence_wake(struct timer_list *t)
0431 {
0432     struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
0433     struct i915_sw_fence *fence;
0434
0435     fence = xchg(&cb->base.fence, NULL);
0436     if (!fence)
0437         return;
0438
0439     pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n",
0440           cb->dma->ops->get_driver_name(cb->dma),
0441           cb->dma->ops->get_timeline_name(cb->dma),
0442           cb->dma->seqno,
0443           i915_sw_fence_debug_hint(fence));
0444
0445     i915_sw_fence_set_error_once(fence, -ETIMEDOUT);
0446     i915_sw_fence_complete(fence);
0447 }
0448
0449 static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
0450                      struct dma_fence_cb *data)
0451 {
0452     struct i915_sw_dma_fence_cb_timer *cb =
0453         container_of(data, typeof(*cb), base.base);
0454     struct i915_sw_fence *fence;
0455
0456     fence = xchg(&cb->base.fence, NULL);
0457     if (fence) {
0458         i915_sw_fence_set_error_once(fence, dma->error);
0459         i915_sw_fence_complete(fence);
0460     }
0461
0462     irq_work_queue(&cb->work);
0463 }
0464
0465 static void irq_i915_sw_fence_work(struct irq_work *wrk)
0466 {
0467     struct i915_sw_dma_fence_cb_timer *cb =
0468         container_of(wrk, typeof(*cb), work);
0469
0470     del_timer_sync(&cb->timer);
0471     dma_fence_put(cb->dma);
0472
0473     kfree_rcu(cb, rcu);
0474 }
0475
0476 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
0477                   struct dma_fence *dma,
0478                   unsigned long timeout,
0479                   gfp_t gfp)
0480 {
0481     struct i915_sw_dma_fence_cb *cb;
0482     dma_fence_func_t func;
0483     int ret;
0484
0485     debug_fence_assert(fence);
0486     might_sleep_if(gfpflags_allow_blocking(gfp));
0487
0488     if (dma_fence_is_signaled(dma)) {
0489         i915_sw_fence_set_error_once(fence, dma->error);
0490         return 0;
0491     }
0492
0493     cb = kmalloc(timeout ?
0494              sizeof(struct i915_sw_dma_fence_cb_timer) :
0495              sizeof(struct i915_sw_dma_fence_cb),
0496              gfp);
0497     if (!cb) {
0498         if (!gfpflags_allow_blocking(gfp))
0499             return -ENOMEM;
0500
0501         ret = dma_fence_wait(dma, false);
0502         if (ret)
0503             return ret;
0504
0505         i915_sw_fence_set_error_once(fence, dma->error);
0506         return 0;
0507     }
0508
0509     cb->fence = fence;
0510     i915_sw_fence_await(fence);
0511
0512     func = dma_i915_sw_fence_wake;
0513     if (timeout) {
0514         struct i915_sw_dma_fence_cb_timer *timer =
0515             container_of(cb, typeof(*timer), base);
0516
0517         timer->dma = dma_fence_get(dma);
0518         init_irq_work(&timer->work, irq_i915_sw_fence_work);
0519
0520         timer_setup(&timer->timer,
0521                 timer_i915_sw_fence_wake, TIMER_IRQSAFE);
0522         mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));
0523
0524         func = dma_i915_sw_fence_wake_timer;
0525     }
0526
0527     ret = dma_fence_add_callback(dma, &cb->base, func);
0528     if (ret == 0) {
0529         ret = 1;
0530     } else {
0531         func(dma, &cb->base);
0532         if (ret == -ENOENT) /* fence already signaled */
0533             ret = 0;
0534     }
0535
0536     return ret;
0537 }
0538
0539 static void __dma_i915_sw_fence_wake(struct dma_fence *dma,
0540                      struct dma_fence_cb *data)
0541 {
0542     struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
0543
0544     i915_sw_fence_set_error_once(cb->fence, dma->error);
0545     i915_sw_fence_complete(cb->fence);
0546 }
0547
0548 int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
0549                     struct dma_fence *dma,
0550                     struct i915_sw_dma_fence_cb *cb)
0551 {
0552     int ret;
0553
0554     debug_fence_assert(fence);
0555
0556     if (dma_fence_is_signaled(dma)) {
0557         i915_sw_fence_set_error_once(fence, dma->error);
0558         return 0;
0559     }
0560
0561     cb->fence = fence;
0562     i915_sw_fence_await(fence);
0563
0564     ret = 1;
0565     if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) {
0566         /* fence already signaled */
0567         __dma_i915_sw_fence_wake(dma, &cb->base);
0568         ret = 0;
0569     }
0570
0571     return ret;
0572 }
0573
0574 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
0575                     struct dma_resv *resv,
0576                     const struct dma_fence_ops *exclude,
0577                     bool write,
0578                     unsigned long timeout,
0579                     gfp_t gfp)
0580 {
0581     struct dma_resv_iter cursor;
0582     struct dma_fence *f;
0583     int ret = 0, pending;
0584
0585     debug_fence_assert(fence);
0586     might_sleep_if(gfpflags_allow_blocking(gfp));
0587
0588     dma_resv_iter_begin(&cursor, resv, dma_resv_usage_rw(write));
0589     dma_resv_for_each_fence_unlocked(&cursor, f) {
0590         pending = i915_sw_fence_await_dma_fence(fence, f, timeout,
0591                             gfp);
0592         if (pending < 0) {
0593             ret = pending;
0594             break;
0595         }
0596
0597         ret |= pending;
0598     }
0599     dma_resv_iter_end(&cursor);
0600     return ret;
0601 }
0602
0603 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
0604 #include "selftests/lib_sw_fence.c"
0605 #include "selftests/i915_sw_fence.c"
0606 #endif