drivers/dma-buf/dma-fence-array.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * dma-fence-array: aggregate fences to be waited together
0004  *
0005  * Copyright (C) 2016 Collabora Ltd
0006  * Copyright (C) 2016 Advanced Micro Devices, Inc.
0007  * Authors:
0008  *  Gustavo Padovan <gustavo@padovan.org>
0009  *  Christian König <christian.koenig@amd.com>
0010  */
0011
0012 #include <linux/export.h>
0013 #include <linux/slab.h>
0014 #include <linux/dma-fence-array.h>
0015
0016 #define PENDING_ERROR 1
0017
0018 static const char *dma_fence_array_get_driver_name(struct dma_fence *fence)
0019 {
0020     return "dma_fence_array";
0021 }
0022
0023 static const char *dma_fence_array_get_timeline_name(struct dma_fence *fence)
0024 {
0025     return "unbound";
0026 }
0027
0028 static void dma_fence_array_set_pending_error(struct dma_fence_array *array,
0029                           int error)
0030 {
0031     /*
0032      * Propagate the first error reported by any of our fences, but only
0033      * before we ourselves are signaled.
0034      */
0035     if (error)
0036         cmpxchg(&array->base.error, PENDING_ERROR, error);
0037 }
0038
0039 static void dma_fence_array_clear_pending_error(struct dma_fence_array *array)
0040 {
0041     /* Clear the error flag if not actually set. */
0042     cmpxchg(&array->base.error, PENDING_ERROR, 0);
0043 }
0044
0045 static void irq_dma_fence_array_work(struct irq_work *wrk)
0046 {
0047     struct dma_fence_array *array = container_of(wrk, typeof(*array), work);
0048
0049     dma_fence_array_clear_pending_error(array);
0050
0051     dma_fence_signal(&array->base);
0052     dma_fence_put(&array->base);
0053 }
0054
0055 static void dma_fence_array_cb_func(struct dma_fence *f,
0056                     struct dma_fence_cb *cb)
0057 {
0058     struct dma_fence_array_cb *array_cb =
0059         container_of(cb, struct dma_fence_array_cb, cb);
0060     struct dma_fence_array *array = array_cb->array;
0061
0062     dma_fence_array_set_pending_error(array, f->error);
0063
0064     if (atomic_dec_and_test(&array->num_pending))
0065         irq_work_queue(&array->work);
0066     else
0067         dma_fence_put(&array->base);
0068 }
0069
0070 static bool dma_fence_array_enable_signaling(struct dma_fence *fence)
0071 {
0072     struct dma_fence_array *array = to_dma_fence_array(fence);
0073     struct dma_fence_array_cb *cb = (void *)(&array[1]);
0074     unsigned i;
0075
0076     for (i = 0; i < array->num_fences; ++i) {
0077         cb[i].array = array;
0078         /*
0079          * As we may report that the fence is signaled before all
0080          * callbacks are complete, we need to take an additional
0081          * reference count on the array so that we do not free it too
0082          * early. The core fence handling will only hold the reference
0083          * until we signal the array as complete (but that is now
0084          * insufficient).
0085          */
0086         dma_fence_get(&array->base);
0087         if (dma_fence_add_callback(array->fences[i], &cb[i].cb,
0088                        dma_fence_array_cb_func)) {
0089             int error = array->fences[i]->error;
0090
0091             dma_fence_array_set_pending_error(array, error);
0092             dma_fence_put(&array->base);
0093             if (atomic_dec_and_test(&array->num_pending)) {
0094                 dma_fence_array_clear_pending_error(array);
0095                 return false;
0096             }
0097         }
0098     }
0099
0100     return true;
0101 }
0102
0103 static bool dma_fence_array_signaled(struct dma_fence *fence)
0104 {
0105     struct dma_fence_array *array = to_dma_fence_array(fence);
0106
0107     if (atomic_read(&array->num_pending) > 0)
0108         return false;
0109
0110     dma_fence_array_clear_pending_error(array);
0111     return true;
0112 }
0113
0114 static void dma_fence_array_release(struct dma_fence *fence)
0115 {
0116     struct dma_fence_array *array = to_dma_fence_array(fence);
0117     unsigned i;
0118
0119     for (i = 0; i < array->num_fences; ++i)
0120         dma_fence_put(array->fences[i]);
0121
0122     kfree(array->fences);
0123     dma_fence_free(fence);
0124 }
0125
0126 const struct dma_fence_ops dma_fence_array_ops = {
0127     .get_driver_name = dma_fence_array_get_driver_name,
0128     .get_timeline_name = dma_fence_array_get_timeline_name,
0129     .enable_signaling = dma_fence_array_enable_signaling,
0130     .signaled = dma_fence_array_signaled,
0131     .release = dma_fence_array_release,
0132 };
0133 EXPORT_SYMBOL(dma_fence_array_ops);
0134
0135 /**
0136  * dma_fence_array_create - Create a custom fence array
0137  * @num_fences:     [in]    number of fences to add in the array
0138  * @fences:     [in]    array containing the fences
0139  * @context:        [in]    fence context to use
0140  * @seqno:      [in]    sequence number to use
0141  * @signal_on_any:  [in]    signal on any fence in the array
0142  *
0143  * Allocate a dma_fence_array object and initialize the base fence with
0144  * dma_fence_init().
0145  * In case of error it returns NULL.
0146  *
0147  * The caller should allocate the fences array with num_fences size
0148  * and fill it with the fences it wants to add to the object. Ownership of this
0149  * array is taken and dma_fence_put() is used on each fence on release.
0150  *
0151  * If @signal_on_any is true the fence array signals if any fence in the array
0152  * signals, otherwise it signals when all fences in the array signal.
0153  */
0154 struct dma_fence_array *dma_fence_array_create(int num_fences,
0155                            struct dma_fence **fences,
0156                            u64 context, unsigned seqno,
0157                            bool signal_on_any)
0158 {
0159     struct dma_fence_array *array;
0160     size_t size = sizeof(*array);
0161
0162     WARN_ON(!num_fences || !fences);
0163
0164     /* Allocate the callback structures behind the array. */
0165     size += num_fences * sizeof(struct dma_fence_array_cb);
0166     array = kzalloc(size, GFP_KERNEL);
0167     if (!array)
0168         return NULL;
0169
0170     spin_lock_init(&array->lock);
0171     dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock,
0172                context, seqno);
0173     init_irq_work(&array->work, irq_dma_fence_array_work);
0174
0175     array->num_fences = num_fences;
0176     atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);
0177     array->fences = fences;
0178
0179     array->base.error = PENDING_ERROR;
0180
0181     /*
0182      * dma_fence_array objects should never contain any other fence
0183      * containers or otherwise we run into recursion and potential kernel
0184      * stack overflow on operations on the dma_fence_array.
0185      *
0186      * The correct way of handling this is to flatten out the array by the
0187      * caller instead.
0188      *
0189      * Enforce this here by checking that we don't create a dma_fence_array
0190      * with any container inside.
0191      */
0192     while (num_fences--)
0193         WARN_ON(dma_fence_is_container(fences[num_fences]));
0194
0195     return array;
0196 }
0197 EXPORT_SYMBOL(dma_fence_array_create);
0198
0199 /**
0200  * dma_fence_match_context - Check if all fences are from the given context
0201  * @fence:      [in]    fence or fence array
0202  * @context:        [in]    fence context to check all fences against
0203  *
0204  * Checks the provided fence or, for a fence array, all fences in the array
0205  * against the given context. Returns false if any fence is from a different
0206  * context.
0207  */
0208 bool dma_fence_match_context(struct dma_fence *fence, u64 context)
0209 {
0210     struct dma_fence_array *array = to_dma_fence_array(fence);
0211     unsigned i;
0212
0213     if (!dma_fence_is_array(fence))
0214         return fence->context == context;
0215
0216     for (i = 0; i < array->num_fences; i++) {
0217         if (array->fences[i]->context != context)
0218             return false;
0219     }
0220
0221     return true;
0222 }
0223 EXPORT_SYMBOL(dma_fence_match_context);
0224
0225 struct dma_fence *dma_fence_array_first(struct dma_fence *head)
0226 {
0227     struct dma_fence_array *array;
0228
0229     if (!head)
0230         return NULL;
0231
0232     array = to_dma_fence_array(head);
0233     if (!array)
0234         return head;
0235
0236     if (!array->num_fences)
0237         return NULL;
0238
0239     return array->fences[0];
0240 }
0241 EXPORT_SYMBOL(dma_fence_array_first);
0242
0243 struct dma_fence *dma_fence_array_next(struct dma_fence *head,
0244                        unsigned int index)
0245 {
0246     struct dma_fence_array *array = to_dma_fence_array(head);
0247
0248     if (!array || index >= array->num_fences)
0249         return NULL;
0250
0251     return array->fences[index];
0252 }
0253 EXPORT_SYMBOL(dma_fence_array_next);