drm/i915/i915_vma_resource.c

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2021 Intel Corporation
0004  */
0005
0006 #include <linux/interval_tree_generic.h>
0007 #include <linux/sched/mm.h>
0008
0009 #include "i915_sw_fence.h"
0010 #include "i915_vma_resource.h"
0011 #include "i915_drv.h"
0012 #include "intel_memory_region.h"
0013
0014 #include "gt/intel_gtt.h"
0015
0016 static struct kmem_cache *slab_vma_resources;
0017
0018 /**
0019  * DOC:
0020  * We use a per-vm interval tree to keep track of vma_resources
0021  * scheduled for unbind but not yet unbound. The tree is protected by
0022  * the vm mutex, and nodes are removed just after the unbind fence signals.
0023  * The removal takes the vm mutex from a kernel thread which we need to
0024  * keep in mind so that we don't grab the mutex and try to wait for all
0025  * pending unbinds to complete, because that will temporaryily block many
0026  * of the workqueue threads, and people will get angry.
0027  *
0028  * We should consider using a single ordered fence per VM instead but that
0029  * requires ordering the unbinds and might introduce unnecessary waiting
0030  * for unrelated unbinds. Amount of code will probably be roughly the same
0031  * due to the simplicity of using the interval tree interface.
0032  *
0033  * Another drawback of this interval tree is that the complexity of insertion
0034  * and removal of fences increases as O(ln(pending_unbinds)) instead of
0035  * O(1) for a single fence without interval tree.
0036  */
0037 #define VMA_RES_START(_node) ((_node)->start)
0038 #define VMA_RES_LAST(_node) ((_node)->start + (_node)->node_size - 1)
0039 INTERVAL_TREE_DEFINE(struct i915_vma_resource, rb,
0040              u64, __subtree_last,
0041              VMA_RES_START, VMA_RES_LAST, static, vma_res_itree);
0042
0043 /* Callbacks for the unbind dma-fence. */
0044
0045 /**
0046  * i915_vma_resource_alloc - Allocate a vma resource
0047  *
0048  * Return: A pointer to a cleared struct i915_vma_resource or
0049  * a -ENOMEM error pointer if allocation fails.
0050  */
0051 struct i915_vma_resource *i915_vma_resource_alloc(void)
0052 {
0053     struct i915_vma_resource *vma_res =
0054         kmem_cache_zalloc(slab_vma_resources, GFP_KERNEL);
0055
0056     return vma_res ? vma_res : ERR_PTR(-ENOMEM);
0057 }
0058
0059 /**
0060  * i915_vma_resource_free - Free a vma resource
0061  * @vma_res: The vma resource to free.
0062  */
0063 void i915_vma_resource_free(struct i915_vma_resource *vma_res)
0064 {
0065     if (vma_res)
0066         kmem_cache_free(slab_vma_resources, vma_res);
0067 }
0068
0069 static const char *get_driver_name(struct dma_fence *fence)
0070 {
0071     return "vma unbind fence";
0072 }
0073
0074 static const char *get_timeline_name(struct dma_fence *fence)
0075 {
0076     return "unbound";
0077 }
0078
0079 static void unbind_fence_free_rcu(struct rcu_head *head)
0080 {
0081     struct i915_vma_resource *vma_res =
0082         container_of(head, typeof(*vma_res), unbind_fence.rcu);
0083
0084     i915_vma_resource_free(vma_res);
0085 }
0086
0087 static void unbind_fence_release(struct dma_fence *fence)
0088 {
0089     struct i915_vma_resource *vma_res =
0090         container_of(fence, typeof(*vma_res), unbind_fence);
0091
0092     i915_sw_fence_fini(&vma_res->chain);
0093
0094     call_rcu(&fence->rcu, unbind_fence_free_rcu);
0095 }
0096
0097 static struct dma_fence_ops unbind_fence_ops = {
0098     .get_driver_name = get_driver_name,
0099     .get_timeline_name = get_timeline_name,
0100     .release = unbind_fence_release,
0101 };
0102
0103 static void __i915_vma_resource_unhold(struct i915_vma_resource *vma_res)
0104 {
0105     struct i915_address_space *vm;
0106
0107     if (!refcount_dec_and_test(&vma_res->hold_count))
0108         return;
0109
0110     dma_fence_signal(&vma_res->unbind_fence);
0111
0112     vm = vma_res->vm;
0113     if (vma_res->wakeref)
0114         intel_runtime_pm_put(&vm->i915->runtime_pm, vma_res->wakeref);
0115
0116     vma_res->vm = NULL;
0117     if (!RB_EMPTY_NODE(&vma_res->rb)) {
0118         mutex_lock(&vm->mutex);
0119         vma_res_itree_remove(vma_res, &vm->pending_unbind);
0120         mutex_unlock(&vm->mutex);
0121     }
0122
0123     if (vma_res->bi.pages_rsgt)
0124         i915_refct_sgt_put(vma_res->bi.pages_rsgt);
0125 }
0126
0127 /**
0128  * i915_vma_resource_unhold - Unhold the signaling of the vma resource unbind
0129  * fence.
0130  * @vma_res: The vma resource.
0131  * @lockdep_cookie: The lockdep cookie returned from i915_vma_resource_hold.
0132  *
0133  * The function may leave a dma_fence critical section.
0134  */
0135 void i915_vma_resource_unhold(struct i915_vma_resource *vma_res,
0136                   bool lockdep_cookie)
0137 {
0138     dma_fence_end_signalling(lockdep_cookie);
0139
0140     if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
0141         unsigned long irq_flags;
0142
0143         /* Inefficient open-coded might_lock_irqsave() */
0144         spin_lock_irqsave(&vma_res->lock, irq_flags);
0145         spin_unlock_irqrestore(&vma_res->lock, irq_flags);
0146     }
0147
0148     __i915_vma_resource_unhold(vma_res);
0149 }
0150
0151 /**
0152  * i915_vma_resource_hold - Hold the signaling of the vma resource unbind fence.
0153  * @vma_res: The vma resource.
0154  * @lockdep_cookie: Pointer to a bool serving as a lockdep cooke that should
0155  * be given as an argument to the pairing i915_vma_resource_unhold.
0156  *
0157  * If returning true, the function enters a dma_fence signalling critical
0158  * section if not in one already.
0159  *
0160  * Return: true if holding successful, false if not.
0161  */
0162 bool i915_vma_resource_hold(struct i915_vma_resource *vma_res,
0163                 bool *lockdep_cookie)
0164 {
0165     bool held = refcount_inc_not_zero(&vma_res->hold_count);
0166
0167     if (held)
0168         *lockdep_cookie = dma_fence_begin_signalling();
0169
0170     return held;
0171 }
0172
0173 static void i915_vma_resource_unbind_work(struct work_struct *work)
0174 {
0175     struct i915_vma_resource *vma_res =
0176         container_of(work, typeof(*vma_res), work);
0177     struct i915_address_space *vm = vma_res->vm;
0178     bool lockdep_cookie;
0179
0180     lockdep_cookie = dma_fence_begin_signalling();
0181     if (likely(!vma_res->skip_pte_rewrite))
0182         vma_res->ops->unbind_vma(vm, vma_res);
0183
0184     dma_fence_end_signalling(lockdep_cookie);
0185     __i915_vma_resource_unhold(vma_res);
0186     i915_vma_resource_put(vma_res);
0187 }
0188
0189 static int
0190 i915_vma_resource_fence_notify(struct i915_sw_fence *fence,
0191                    enum i915_sw_fence_notify state)
0192 {
0193     struct i915_vma_resource *vma_res =
0194         container_of(fence, typeof(*vma_res), chain);
0195     struct dma_fence *unbind_fence =
0196         &vma_res->unbind_fence;
0197
0198     switch (state) {
0199     case FENCE_COMPLETE:
0200         dma_fence_get(unbind_fence);
0201         if (vma_res->immediate_unbind) {
0202             i915_vma_resource_unbind_work(&vma_res->work);
0203         } else {
0204             INIT_WORK(&vma_res->work, i915_vma_resource_unbind_work);
0205             queue_work(system_unbound_wq, &vma_res->work);
0206         }
0207         break;
0208     case FENCE_FREE:
0209         i915_vma_resource_put(vma_res);
0210         break;
0211     }
0212
0213     return NOTIFY_DONE;
0214 }
0215
0216 /**
0217  * i915_vma_resource_unbind - Unbind a vma resource
0218  * @vma_res: The vma resource to unbind.
0219  *
0220  * At this point this function does little more than publish a fence that
0221  * signals immediately unless signaling is held back.
0222  *
0223  * Return: A refcounted pointer to a dma-fence that signals when unbinding is
0224  * complete.
0225  */
0226 struct dma_fence *i915_vma_resource_unbind(struct i915_vma_resource *vma_res,
0227                        u32 *tlb)
0228 {
0229     struct i915_address_space *vm = vma_res->vm;
0230
0231     vma_res->tlb = tlb;
0232
0233     /* Reference for the sw fence */
0234     i915_vma_resource_get(vma_res);
0235
0236     /* Caller must already have a wakeref in this case. */
0237     if (vma_res->needs_wakeref)
0238         vma_res->wakeref = intel_runtime_pm_get_if_in_use(&vm->i915->runtime_pm);
0239
0240     if (atomic_read(&vma_res->chain.pending) <= 1) {
0241         RB_CLEAR_NODE(&vma_res->rb);
0242         vma_res->immediate_unbind = 1;
0243     } else {
0244         vma_res_itree_insert(vma_res, &vma_res->vm->pending_unbind);
0245     }
0246
0247     i915_sw_fence_commit(&vma_res->chain);
0248
0249     return &vma_res->unbind_fence;
0250 }
0251
0252 /**
0253  * __i915_vma_resource_init - Initialize a vma resource.
0254  * @vma_res: The vma resource to initialize
0255  *
0256  * Initializes the private members of a vma resource.
0257  */
0258 void __i915_vma_resource_init(struct i915_vma_resource *vma_res)
0259 {
0260     spin_lock_init(&vma_res->lock);
0261     dma_fence_init(&vma_res->unbind_fence, &unbind_fence_ops,
0262                &vma_res->lock, 0, 0);
0263     refcount_set(&vma_res->hold_count, 1);
0264     i915_sw_fence_init(&vma_res->chain, i915_vma_resource_fence_notify);
0265 }
0266
0267 static void
0268 i915_vma_resource_color_adjust_range(struct i915_address_space *vm,
0269                      u64 *start,
0270                      u64 *end)
0271 {
0272     if (i915_vm_has_cache_coloring(vm)) {
0273         if (*start)
0274             *start -= I915_GTT_PAGE_SIZE;
0275         *end += I915_GTT_PAGE_SIZE;
0276     }
0277 }
0278
0279 /**
0280  * i915_vma_resource_bind_dep_sync - Wait for / sync all unbinds touching a
0281  * certain vm range.
0282  * @vm: The vm to look at.
0283  * @offset: The range start.
0284  * @size: The range size.
0285  * @intr: Whether to wait interrubtible.
0286  *
0287  * The function needs to be called with the vm lock held.
0288  *
0289  * Return: Zero on success, -ERESTARTSYS if interrupted and @intr==true
0290  */
0291 int i915_vma_resource_bind_dep_sync(struct i915_address_space *vm,
0292                     u64 offset,
0293                     u64 size,
0294                     bool intr)
0295 {
0296     struct i915_vma_resource *node;
0297     u64 last = offset + size - 1;
0298
0299     lockdep_assert_held(&vm->mutex);
0300     might_sleep();
0301
0302     i915_vma_resource_color_adjust_range(vm, &offset, &last);
0303     node = vma_res_itree_iter_first(&vm->pending_unbind, offset, last);
0304     while (node) {
0305         int ret = dma_fence_wait(&node->unbind_fence, intr);
0306
0307         if (ret)
0308             return ret;
0309
0310         node = vma_res_itree_iter_next(node, offset, last);
0311     }
0312
0313     return 0;
0314 }
0315
0316 /**
0317  * i915_vma_resource_bind_dep_sync_all - Wait for / sync all unbinds of a vm,
0318  * releasing the vm lock while waiting.
0319  * @vm: The vm to look at.
0320  *
0321  * The function may not be called with the vm lock held.
0322  * Typically this is called at vm destruction to finish any pending
0323  * unbind operations. The vm mutex is released while waiting to avoid
0324  * stalling kernel workqueues trying to grab the mutex.
0325  */
0326 void i915_vma_resource_bind_dep_sync_all(struct i915_address_space *vm)
0327 {
0328     struct i915_vma_resource *node;
0329     struct dma_fence *fence;
0330
0331     do {
0332         fence = NULL;
0333         mutex_lock(&vm->mutex);
0334         node = vma_res_itree_iter_first(&vm->pending_unbind, 0,
0335                         U64_MAX);
0336         if (node)
0337             fence = dma_fence_get_rcu(&node->unbind_fence);
0338         mutex_unlock(&vm->mutex);
0339
0340         if (fence) {
0341             /*
0342              * The wait makes sure the node eventually removes
0343              * itself from the tree.
0344              */
0345             dma_fence_wait(fence, false);
0346             dma_fence_put(fence);
0347         }
0348     } while (node);
0349 }
0350
0351 /**
0352  * i915_vma_resource_bind_dep_await - Have a struct i915_sw_fence await all
0353  * pending unbinds in a certain range of a vm.
0354  * @vm: The vm to look at.
0355  * @sw_fence: The struct i915_sw_fence that will be awaiting the unbinds.
0356  * @offset: The range start.
0357  * @size: The range size.
0358  * @intr: Whether to wait interrubtible.
0359  * @gfp: Allocation mode for memory allocations.
0360  *
0361  * The function makes @sw_fence await all pending unbinds in a certain
0362  * vm range before calling the complete notifier. To be able to await
0363  * each individual unbind, the function needs to allocate memory using
0364  * the @gpf allocation mode. If that fails, the function will instead
0365  * wait for the unbind fence to signal, using @intr to judge whether to
0366  * wait interruptible or not. Note that @gfp should ideally be selected so
0367  * as to avoid any expensive memory allocation stalls and rather fail and
0368  * synchronize itself. For now the vm mutex is required when calling this
0369  * function with means that @gfp can't call into direct reclaim. In reality
0370  * this means that during heavy memory pressure, we will sync in this
0371  * function.
0372  *
0373  * Return: Zero on success, -ERESTARTSYS if interrupted and @intr==true
0374  */
0375 int i915_vma_resource_bind_dep_await(struct i915_address_space *vm,
0376                      struct i915_sw_fence *sw_fence,
0377                      u64 offset,
0378                      u64 size,
0379                      bool intr,
0380                      gfp_t gfp)
0381 {
0382     struct i915_vma_resource *node;
0383     u64 last = offset + size - 1;
0384
0385     lockdep_assert_held(&vm->mutex);
0386     might_alloc(gfp);
0387     might_sleep();
0388
0389     i915_vma_resource_color_adjust_range(vm, &offset, &last);
0390     node = vma_res_itree_iter_first(&vm->pending_unbind, offset, last);
0391     while (node) {
0392         int ret;
0393
0394         ret = i915_sw_fence_await_dma_fence(sw_fence,
0395                             &node->unbind_fence,
0396                             0, gfp);
0397         if (ret < 0) {
0398             ret = dma_fence_wait(&node->unbind_fence, intr);
0399             if (ret)
0400                 return ret;
0401         }
0402
0403         node = vma_res_itree_iter_next(node, offset, last);
0404     }
0405
0406     return 0;
0407 }
0408
0409 void i915_vma_resource_module_exit(void)
0410 {
0411     kmem_cache_destroy(slab_vma_resources);
0412 }
0413
0414 int __init i915_vma_resource_module_init(void)
0415 {
0416     slab_vma_resources = KMEM_CACHE(i915_vma_resource, SLAB_HWCACHE_ALIGN);
0417     if (!slab_vma_resources)
0418         return -ENOMEM;
0419
0420     return 0;
0421 }