i915/gem/i915_gem_tiling.c

0001 /*
0002  * SPDX-License-Identifier: MIT
0003  *
0004  * Copyright © 2008 Intel Corporation
0005  */
0006
0007 #include <linux/string.h>
0008 #include <linux/bitops.h>
0009
0010 #include "i915_drv.h"
0011 #include "i915_gem.h"
0012 #include "i915_gem_ioctls.h"
0013 #include "i915_gem_mman.h"
0014 #include "i915_gem_object.h"
0015 #include "i915_gem_tiling.h"
0016 #include "i915_reg.h"
0017
0018 /**
0019  * DOC: buffer object tiling
0020  *
0021  * i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace
0022  * interface to declare fence register requirements.
0023  *
0024  * In principle GEM doesn't care at all about the internal data layout of an
0025  * object, and hence it also doesn't care about tiling or swizzling. There's two
0026  * exceptions:
0027  *
0028  * - For X and Y tiling the hardware provides detilers for CPU access, so called
0029  *   fences. Since there's only a limited amount of them the kernel must manage
0030  *   these, and therefore userspace must tell the kernel the object tiling if it
0031  *   wants to use fences for detiling.
0032  * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which
0033  *   depends upon the physical page frame number. When swapping such objects the
0034  *   page frame number might change and the kernel must be able to fix this up
0035  *   and hence now the tiling. Note that on a subset of platforms with
0036  *   asymmetric memory channel population the swizzling pattern changes in an
0037  *   unknown way, and for those the kernel simply forbids swapping completely.
0038  *
0039  * Since neither of this applies for new tiling layouts on modern platforms like
0040  * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled.
0041  * Anything else can be handled in userspace entirely without the kernel's
0042  * invovlement.
0043  */
0044
0045 /**
0046  * i915_gem_fence_size - required global GTT size for a fence
0047  * @i915: i915 device
0048  * @size: object size
0049  * @tiling: tiling mode
0050  * @stride: tiling stride
0051  *
0052  * Return the required global GTT size for a fence (view of a tiled object),
0053  * taking into account potential fence register mapping.
0054  */
0055 u32 i915_gem_fence_size(struct drm_i915_private *i915,
0056             u32 size, unsigned int tiling, unsigned int stride)
0057 {
0058     u32 ggtt_size;
0059
0060     GEM_BUG_ON(!size);
0061
0062     if (tiling == I915_TILING_NONE)
0063         return size;
0064
0065     GEM_BUG_ON(!stride);
0066
0067     if (GRAPHICS_VER(i915) >= 4) {
0068         stride *= i915_gem_tile_height(tiling);
0069         GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE));
0070         return roundup(size, stride);
0071     }
0072
0073     /* Previous chips need a power-of-two fence region when tiling */
0074     if (GRAPHICS_VER(i915) == 3)
0075         ggtt_size = 1024*1024;
0076     else
0077         ggtt_size = 512*1024;
0078
0079     while (ggtt_size < size)
0080         ggtt_size <<= 1;
0081
0082     return ggtt_size;
0083 }
0084
0085 /**
0086  * i915_gem_fence_alignment - required global GTT alignment for a fence
0087  * @i915: i915 device
0088  * @size: object size
0089  * @tiling: tiling mode
0090  * @stride: tiling stride
0091  *
0092  * Return the required global GTT alignment for a fence (a view of a tiled
0093  * object), taking into account potential fence register mapping.
0094  */
0095 u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size,
0096                  unsigned int tiling, unsigned int stride)
0097 {
0098     GEM_BUG_ON(!size);
0099
0100     /*
0101      * Minimum alignment is 4k (GTT page size), but might be greater
0102      * if a fence register is needed for the object.
0103      */
0104     if (tiling == I915_TILING_NONE)
0105         return I915_GTT_MIN_ALIGNMENT;
0106
0107     if (GRAPHICS_VER(i915) >= 4)
0108         return I965_FENCE_PAGE;
0109
0110     /*
0111      * Previous chips need to be aligned to the size of the smallest
0112      * fence register that can contain the object.
0113      */
0114     return i915_gem_fence_size(i915, size, tiling, stride);
0115 }
0116
0117 /* Check pitch constraints for all chips & tiling formats */
0118 static bool
0119 i915_tiling_ok(struct drm_i915_gem_object *obj,
0120            unsigned int tiling, unsigned int stride)
0121 {
0122     struct drm_i915_private *i915 = to_i915(obj->base.dev);
0123     unsigned int tile_width;
0124
0125     /* Linear is always fine */
0126     if (tiling == I915_TILING_NONE)
0127         return true;
0128
0129     if (tiling > I915_TILING_LAST)
0130         return false;
0131
0132     /* check maximum stride & object size */
0133     /* i965+ stores the end address of the gtt mapping in the fence
0134      * reg, so dont bother to check the size */
0135     if (GRAPHICS_VER(i915) >= 7) {
0136         if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
0137             return false;
0138     } else if (GRAPHICS_VER(i915) >= 4) {
0139         if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
0140             return false;
0141     } else {
0142         if (stride > 8192)
0143             return false;
0144
0145         if (!is_power_of_2(stride))
0146             return false;
0147     }
0148
0149     if (GRAPHICS_VER(i915) == 2 ||
0150         (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915)))
0151         tile_width = 128;
0152     else
0153         tile_width = 512;
0154
0155     if (!stride || !IS_ALIGNED(stride, tile_width))
0156         return false;
0157
0158     return true;
0159 }
0160
0161 static bool i915_vma_fence_prepare(struct i915_vma *vma,
0162                    int tiling_mode, unsigned int stride)
0163 {
0164     struct drm_i915_private *i915 = vma->vm->i915;
0165     u32 size, alignment;
0166
0167     if (!i915_vma_is_map_and_fenceable(vma))
0168         return true;
0169
0170     size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride);
0171     if (vma->node.size < size)
0172         return false;
0173
0174     alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride);
0175     if (!IS_ALIGNED(vma->node.start, alignment))
0176         return false;
0177
0178     return true;
0179 }
0180
0181 /* Make the current GTT allocation valid for the change in tiling. */
0182 static int
0183 i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
0184                   int tiling_mode, unsigned int stride)
0185 {
0186     struct drm_i915_private *i915 = to_i915(obj->base.dev);
0187     struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
0188     struct i915_vma *vma, *vn;
0189     LIST_HEAD(unbind);
0190     int ret = 0;
0191
0192     if (tiling_mode == I915_TILING_NONE)
0193         return 0;
0194
0195     mutex_lock(&ggtt->vm.mutex);
0196
0197     spin_lock(&obj->vma.lock);
0198     for_each_ggtt_vma(vma, obj) {
0199         GEM_BUG_ON(vma->vm != &ggtt->vm);
0200
0201         if (i915_vma_fence_prepare(vma, tiling_mode, stride))
0202             continue;
0203
0204         list_move(&vma->vm_link, &unbind);
0205     }
0206     spin_unlock(&obj->vma.lock);
0207
0208     list_for_each_entry_safe(vma, vn, &unbind, vm_link) {
0209         ret = __i915_vma_unbind(vma);
0210         if (ret) {
0211             /* Restore the remaining vma on an error */
0212             list_splice(&unbind, &ggtt->vm.bound_list);
0213             break;
0214         }
0215     }
0216
0217     mutex_unlock(&ggtt->vm.mutex);
0218
0219     return ret;
0220 }
0221
0222 bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
0223 {
0224     struct drm_i915_private *i915 = to_i915(obj->base.dev);
0225
0226     return to_gt(i915)->ggtt->bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
0227         i915_gem_object_is_tiled(obj);
0228 }
0229
0230 int
0231 i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
0232                unsigned int tiling, unsigned int stride)
0233 {
0234     struct drm_i915_private *i915 = to_i915(obj->base.dev);
0235     struct i915_vma *vma;
0236     int err;
0237
0238     /* Make sure we don't cross-contaminate obj->tiling_and_stride */
0239     BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);
0240
0241     GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
0242     GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
0243
0244     if ((tiling | stride) == obj->tiling_and_stride)
0245         return 0;
0246
0247     if (i915_gem_object_is_framebuffer(obj))
0248         return -EBUSY;
0249
0250     /* We need to rebind the object if its current allocation
0251      * no longer meets the alignment restrictions for its new
0252      * tiling mode. Otherwise we can just leave it alone, but
0253      * need to ensure that any fence register is updated before
0254      * the next fenced (either through the GTT or by the BLT unit
0255      * on older GPUs) access.
0256      *
0257      * After updating the tiling parameters, we then flag whether
0258      * we need to update an associated fence register. Note this
0259      * has to also include the unfenced register the GPU uses
0260      * whilst executing a fenced command for an untiled object.
0261      */
0262
0263     i915_gem_object_lock(obj, NULL);
0264     if (i915_gem_object_is_framebuffer(obj)) {
0265         i915_gem_object_unlock(obj);
0266         return -EBUSY;
0267     }
0268
0269     err = i915_gem_object_fence_prepare(obj, tiling, stride);
0270     if (err) {
0271         i915_gem_object_unlock(obj);
0272         return err;
0273     }
0274
0275     /* If the memory has unknown (i.e. varying) swizzling, we pin the
0276      * pages to prevent them being swapped out and causing corruption
0277      * due to the change in swizzling.
0278      */
0279     if (i915_gem_object_has_pages(obj) &&
0280         obj->mm.madv == I915_MADV_WILLNEED &&
0281         i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
0282         if (tiling == I915_TILING_NONE) {
0283             GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
0284             i915_gem_object_clear_tiling_quirk(obj);
0285             i915_gem_object_make_shrinkable(obj);
0286         }
0287         if (!i915_gem_object_is_tiled(obj)) {
0288             GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
0289             i915_gem_object_make_unshrinkable(obj);
0290             i915_gem_object_set_tiling_quirk(obj);
0291         }
0292     }
0293
0294     spin_lock(&obj->vma.lock);
0295     for_each_ggtt_vma(vma, obj) {
0296         vma->fence_size =
0297             i915_gem_fence_size(i915, vma->size, tiling, stride);
0298         vma->fence_alignment =
0299             i915_gem_fence_alignment(i915,
0300                          vma->size, tiling, stride);
0301
0302         if (vma->fence)
0303             vma->fence->dirty = true;
0304     }
0305     spin_unlock(&obj->vma.lock);
0306
0307     obj->tiling_and_stride = tiling | stride;
0308     i915_gem_object_unlock(obj);
0309
0310     /* Force the fence to be reacquired for GTT access */
0311     i915_gem_object_release_mmap_gtt(obj);
0312
0313     /* Try to preallocate memory required to save swizzling on put-pages */
0314     if (i915_gem_object_needs_bit17_swizzle(obj)) {
0315         if (!obj->bit_17) {
0316             obj->bit_17 = bitmap_zalloc(obj->base.size >> PAGE_SHIFT,
0317                             GFP_KERNEL);
0318         }
0319     } else {
0320         bitmap_free(obj->bit_17);
0321         obj->bit_17 = NULL;
0322     }
0323
0324     return 0;
0325 }
0326
0327 /**
0328  * i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode
0329  * @dev: DRM device
0330  * @data: data pointer for the ioctl
0331  * @file: DRM file for the ioctl call
0332  *
0333  * Sets the tiling mode of an object, returning the required swizzling of
0334  * bit 6 of addresses in the object.
0335  *
0336  * Called by the user via ioctl.
0337  *
0338  * Returns:
0339  * Zero on success, negative errno on failure.
0340  */
0341 int
0342 i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
0343               struct drm_file *file)
0344 {
0345     struct drm_i915_private *dev_priv = to_i915(dev);
0346     struct drm_i915_gem_set_tiling *args = data;
0347     struct drm_i915_gem_object *obj;
0348     int err;
0349
0350     if (!to_gt(dev_priv)->ggtt->num_fences)
0351         return -EOPNOTSUPP;
0352
0353     obj = i915_gem_object_lookup(file, args->handle);
0354     if (!obj)
0355         return -ENOENT;
0356
0357     /*
0358      * The tiling mode of proxy objects is handled by its generator, and
0359      * not allowed to be changed by userspace.
0360      */
0361     if (i915_gem_object_is_proxy(obj)) {
0362         err = -ENXIO;
0363         goto err;
0364     }
0365
0366     if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
0367         err = -EINVAL;
0368         goto err;
0369     }
0370
0371     if (args->tiling_mode == I915_TILING_NONE) {
0372         args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
0373         args->stride = 0;
0374     } else {
0375         if (args->tiling_mode == I915_TILING_X)
0376             args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x;
0377         else
0378             args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y;
0379
0380         /* Hide bit 17 swizzling from the user.  This prevents old Mesa
0381          * from aborting the application on sw fallbacks to bit 17,
0382          * and we use the pread/pwrite bit17 paths to swizzle for it.
0383          * If there was a user that was relying on the swizzle
0384          * information for drm_intel_bo_map()ed reads/writes this would
0385          * break it, but we don't have any of those.
0386          */
0387         if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
0388             args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
0389         if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
0390             args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
0391
0392         /* If we can't handle the swizzling, make it untiled. */
0393         if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
0394             args->tiling_mode = I915_TILING_NONE;
0395             args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
0396             args->stride = 0;
0397         }
0398     }
0399
0400     err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
0401
0402     /* We have to maintain this existing ABI... */
0403     args->stride = i915_gem_object_get_stride(obj);
0404     args->tiling_mode = i915_gem_object_get_tiling(obj);
0405
0406 err:
0407     i915_gem_object_put(obj);
0408     return err;
0409 }
0410
0411 /**
0412  * i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode
0413  * @dev: DRM device
0414  * @data: data pointer for the ioctl
0415  * @file: DRM file for the ioctl call
0416  *
0417  * Returns the current tiling mode and required bit 6 swizzling for the object.
0418  *
0419  * Called by the user via ioctl.
0420  *
0421  * Returns:
0422  * Zero on success, negative errno on failure.
0423  */
0424 int
0425 i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
0426               struct drm_file *file)
0427 {
0428     struct drm_i915_gem_get_tiling *args = data;
0429     struct drm_i915_private *dev_priv = to_i915(dev);
0430     struct drm_i915_gem_object *obj;
0431     int err = -ENOENT;
0432
0433     if (!to_gt(dev_priv)->ggtt->num_fences)
0434         return -EOPNOTSUPP;
0435
0436     rcu_read_lock();
0437     obj = i915_gem_object_lookup_rcu(file, args->handle);
0438     if (obj) {
0439         args->tiling_mode =
0440             READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
0441         err = 0;
0442     }
0443     rcu_read_unlock();
0444     if (unlikely(err))
0445         return err;
0446
0447     switch (args->tiling_mode) {
0448     case I915_TILING_X:
0449         args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x;
0450         break;
0451     case I915_TILING_Y:
0452         args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y;
0453         break;
0454     default:
0455     case I915_TILING_NONE:
0456         args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
0457         break;
0458     }
0459
0460     /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
0461     if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
0462         args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
0463     else
0464         args->phys_swizzle_mode = args->swizzle_mode;
0465     if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
0466         args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
0467     if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
0468         args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
0469
0470     return 0;
0471 }