drm/i915/i915_gem_gtt.c

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2010 Daniel Vetter
0004  * Copyright © 2020 Intel Corporation
0005  */
0006
0007 #include <linux/slab.h> /* fault-inject.h is not standalone! */
0008
0009 #include <linux/fault-inject.h>
0010 #include <linux/log2.h>
0011 #include <linux/random.h>
0012 #include <linux/seq_file.h>
0013 #include <linux/stop_machine.h>
0014
0015 #include <asm/set_memory.h>
0016 #include <asm/smp.h>
0017
0018 #include "display/intel_frontbuffer.h"
0019 #include "gt/intel_gt.h"
0020 #include "gt/intel_gt_requests.h"
0021
0022 #include "i915_drv.h"
0023 #include "i915_gem_evict.h"
0024 #include "i915_scatterlist.h"
0025 #include "i915_trace.h"
0026 #include "i915_vgpu.h"
0027
0028 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
0029                    struct sg_table *pages)
0030 {
0031     do {
0032         if (dma_map_sg_attrs(obj->base.dev->dev,
0033                      pages->sgl, pages->nents,
0034                      DMA_BIDIRECTIONAL,
0035                      DMA_ATTR_SKIP_CPU_SYNC |
0036                      DMA_ATTR_NO_KERNEL_MAPPING |
0037                      DMA_ATTR_NO_WARN))
0038             return 0;
0039
0040         /*
0041          * If the DMA remap fails, one cause can be that we have
0042          * too many objects pinned in a small remapping table,
0043          * such as swiotlb. Incrementally purge all other objects and
0044          * try again - if there are no more pages to remove from
0045          * the DMA remapper, i915_gem_shrink will return 0.
0046          */
0047         GEM_BUG_ON(obj->mm.pages == pages);
0048     } while (i915_gem_shrink(NULL, to_i915(obj->base.dev),
0049                  obj->base.size >> PAGE_SHIFT, NULL,
0050                  I915_SHRINK_BOUND |
0051                  I915_SHRINK_UNBOUND));
0052
0053     return -ENOSPC;
0054 }
0055
0056 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
0057                    struct sg_table *pages)
0058 {
0059     struct drm_i915_private *i915 = to_i915(obj->base.dev);
0060     struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
0061
0062     /* XXX This does not prevent more requests being submitted! */
0063     if (unlikely(ggtt->do_idle_maps))
0064         /* Wait a bit, in the hope it avoids the hang */
0065         usleep_range(100, 250);
0066
0067     dma_unmap_sg(i915->drm.dev, pages->sgl, pages->nents,
0068              DMA_BIDIRECTIONAL);
0069 }
0070
0071 /**
0072  * i915_gem_gtt_reserve - reserve a node in an address_space (GTT)
0073  * @vm: the &struct i915_address_space
0074  * @ww: An optional struct i915_gem_ww_ctx.
0075  * @node: the &struct drm_mm_node (typically i915_vma.mode)
0076  * @size: how much space to allocate inside the GTT,
0077  *        must be #I915_GTT_PAGE_SIZE aligned
0078  * @offset: where to insert inside the GTT,
0079  *          must be #I915_GTT_MIN_ALIGNMENT aligned, and the node
0080  *          (@offset + @size) must fit within the address space
0081  * @color: color to apply to node, if this node is not from a VMA,
0082  *         color must be #I915_COLOR_UNEVICTABLE
0083  * @flags: control search and eviction behaviour
0084  *
0085  * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside
0086  * the address space (using @size and @color). If the @node does not fit, it
0087  * tries to evict any overlapping nodes from the GTT, including any
0088  * neighbouring nodes if the colors do not match (to ensure guard pages between
0089  * differing domains). See i915_gem_evict_for_node() for the gory details
0090  * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on
0091  * evicting active overlapping objects, and any overlapping node that is pinned
0092  * or marked as unevictable will also result in failure.
0093  *
0094  * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
0095  * asked to wait for eviction and interrupted.
0096  */
0097 int i915_gem_gtt_reserve(struct i915_address_space *vm,
0098              struct i915_gem_ww_ctx *ww,
0099              struct drm_mm_node *node,
0100              u64 size, u64 offset, unsigned long color,
0101              unsigned int flags)
0102 {
0103     int err;
0104
0105     GEM_BUG_ON(!size);
0106     GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
0107     GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
0108     GEM_BUG_ON(range_overflows(offset, size, vm->total));
0109     GEM_BUG_ON(vm == &to_gt(vm->i915)->ggtt->alias->vm);
0110     GEM_BUG_ON(drm_mm_node_allocated(node));
0111
0112     node->size = size;
0113     node->start = offset;
0114     node->color = color;
0115
0116     err = drm_mm_reserve_node(&vm->mm, node);
0117     if (err != -ENOSPC)
0118         return err;
0119
0120     if (flags & PIN_NOEVICT)
0121         return -ENOSPC;
0122
0123     err = i915_gem_evict_for_node(vm, ww, node, flags);
0124     if (err == 0)
0125         err = drm_mm_reserve_node(&vm->mm, node);
0126
0127     return err;
0128 }
0129
0130 static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
0131 {
0132     u64 range, addr;
0133
0134     GEM_BUG_ON(range_overflows(start, len, end));
0135     GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));
0136
0137     range = round_down(end - len, align) - round_up(start, align);
0138     if (range) {
0139         if (sizeof(unsigned long) == sizeof(u64)) {
0140             addr = get_random_long();
0141         } else {
0142             addr = get_random_int();
0143             if (range > U32_MAX) {
0144                 addr <<= 32;
0145                 addr |= get_random_int();
0146             }
0147         }
0148         div64_u64_rem(addr, range, &addr);
0149         start += addr;
0150     }
0151
0152     return round_up(start, align);
0153 }
0154
0155 /**
0156  * i915_gem_gtt_insert - insert a node into an address_space (GTT)
0157  * @vm: the &struct i915_address_space
0158  * @ww: An optional struct i915_gem_ww_ctx.
0159  * @node: the &struct drm_mm_node (typically i915_vma.node)
0160  * @size: how much space to allocate inside the GTT,
0161  *        must be #I915_GTT_PAGE_SIZE aligned
0162  * @alignment: required alignment of starting offset, may be 0 but
0163  *             if specified, this must be a power-of-two and at least
0164  *             #I915_GTT_MIN_ALIGNMENT
0165  * @color: color to apply to node
0166  * @start: start of any range restriction inside GTT (0 for all),
0167  *         must be #I915_GTT_PAGE_SIZE aligned
0168  * @end: end of any range restriction inside GTT (U64_MAX for all),
0169  *       must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX
0170  * @flags: control search and eviction behaviour
0171  *
0172  * i915_gem_gtt_insert() first searches for an available hole into which
0173  * is can insert the node. The hole address is aligned to @alignment and
0174  * its @size must then fit entirely within the [@start, @end] bounds. The
0175  * nodes on either side of the hole must match @color, or else a guard page
0176  * will be inserted between the two nodes (or the node evicted). If no
0177  * suitable hole is found, first a victim is randomly selected and tested
0178  * for eviction, otherwise then the LRU list of objects within the GTT
0179  * is scanned to find the first set of replacement nodes to create the hole.
0180  * Those old overlapping nodes are evicted from the GTT (and so must be
0181  * rebound before any future use). Any node that is currently pinned cannot
0182  * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently
0183  * active and #PIN_NONBLOCK is specified, that node is also skipped when
0184  * searching for an eviction candidate. See i915_gem_evict_something() for
0185  * the gory details on the eviction algorithm.
0186  *
0187  * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
0188  * asked to wait for eviction and interrupted.
0189  */
0190 int i915_gem_gtt_insert(struct i915_address_space *vm,
0191             struct i915_gem_ww_ctx *ww,
0192             struct drm_mm_node *node,
0193             u64 size, u64 alignment, unsigned long color,
0194             u64 start, u64 end, unsigned int flags)
0195 {
0196     enum drm_mm_insert_mode mode;
0197     u64 offset;
0198     int err;
0199
0200     lockdep_assert_held(&vm->mutex);
0201
0202     GEM_BUG_ON(!size);
0203     GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
0204     GEM_BUG_ON(alignment && !is_power_of_2(alignment));
0205     GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
0206     GEM_BUG_ON(start >= end);
0207     GEM_BUG_ON(start > 0  && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
0208     GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
0209     GEM_BUG_ON(vm == &to_gt(vm->i915)->ggtt->alias->vm);
0210     GEM_BUG_ON(drm_mm_node_allocated(node));
0211
0212     if (unlikely(range_overflows(start, size, end)))
0213         return -ENOSPC;
0214
0215     if (unlikely(round_up(start, alignment) > round_down(end - size, alignment)))
0216         return -ENOSPC;
0217
0218     mode = DRM_MM_INSERT_BEST;
0219     if (flags & PIN_HIGH)
0220         mode = DRM_MM_INSERT_HIGHEST;
0221     if (flags & PIN_MAPPABLE)
0222         mode = DRM_MM_INSERT_LOW;
0223
0224     /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
0225      * so we know that we always have a minimum alignment of 4096.
0226      * The drm_mm range manager is optimised to return results
0227      * with zero alignment, so where possible use the optimal
0228      * path.
0229      */
0230     BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE);
0231     if (alignment <= I915_GTT_MIN_ALIGNMENT)
0232         alignment = 0;
0233
0234     err = drm_mm_insert_node_in_range(&vm->mm, node,
0235                       size, alignment, color,
0236                       start, end, mode);
0237     if (err != -ENOSPC)
0238         return err;
0239
0240     if (mode & DRM_MM_INSERT_ONCE) {
0241         err = drm_mm_insert_node_in_range(&vm->mm, node,
0242                           size, alignment, color,
0243                           start, end,
0244                           DRM_MM_INSERT_BEST);
0245         if (err != -ENOSPC)
0246             return err;
0247     }
0248
0249     if (flags & PIN_NOEVICT)
0250         return -ENOSPC;
0251
0252     /*
0253      * No free space, pick a slot at random.
0254      *
0255      * There is a pathological case here using a GTT shared between
0256      * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt):
0257      *
0258      *    |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->|
0259      *         (64k objects)             (448k objects)
0260      *
0261      * Now imagine that the eviction LRU is ordered top-down (just because
0262      * pathology meets real life), and that we need to evict an object to
0263      * make room inside the aperture. The eviction scan then has to walk
0264      * the 448k list before it finds one within range. And now imagine that
0265      * it has to search for a new hole between every byte inside the memcpy,
0266      * for several simultaneous clients.
0267      *
0268      * On a full-ppgtt system, if we have run out of available space, there
0269      * will be lots and lots of objects in the eviction list! Again,
0270      * searching that LRU list may be slow if we are also applying any
0271      * range restrictions (e.g. restriction to low 4GiB) and so, for
0272      * simplicity and similarilty between different GTT, try the single
0273      * random replacement first.
0274      */
0275     offset = random_offset(start, end,
0276                    size, alignment ?: I915_GTT_MIN_ALIGNMENT);
0277     err = i915_gem_gtt_reserve(vm, ww, node, size, offset, color, flags);
0278     if (err != -ENOSPC)
0279         return err;
0280
0281     if (flags & PIN_NOSEARCH)
0282         return -ENOSPC;
0283
0284     /* Randomly selected placement is pinned, do a search */
0285     err = i915_gem_evict_something(vm, ww, size, alignment, color,
0286                        start, end, flags);
0287     if (err)
0288         return err;
0289
0290     return drm_mm_insert_node_in_range(&vm->mm, node,
0291                        size, alignment, color,
0292                        start, end, DRM_MM_INSERT_EVICT);
0293 }
0294
0295 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
0296 #include "selftests/i915_gem_gtt.c"
0297 #endif