i915/gem/i915_gem_userptr.c

0001 /*
0002  * SPDX-License-Identifier: MIT
0003  *
0004  * Copyright © 2012-2014 Intel Corporation
0005  *
0006   * Based on amdgpu_mn, which bears the following notice:
0007  *
0008  * Copyright 2014 Advanced Micro Devices, Inc.
0009  * All Rights Reserved.
0010  *
0011  * Permission is hereby granted, free of charge, to any person obtaining a
0012  * copy of this software and associated documentation files (the
0013  * "Software"), to deal in the Software without restriction, including
0014  * without limitation the rights to use, copy, modify, merge, publish,
0015  * distribute, sub license, and/or sell copies of the Software, and to
0016  * permit persons to whom the Software is furnished to do so, subject to
0017  * the following conditions:
0018  *
0019  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0020  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0021  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
0022  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
0023  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
0024  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
0025  * USE OR OTHER DEALINGS IN THE SOFTWARE.
0026  *
0027  * The above copyright notice and this permission notice (including the
0028  * next paragraph) shall be included in all copies or substantial portions
0029  * of the Software.
0030  *
0031  */
0032 /*
0033  * Authors:
0034  *    Christian König <christian.koenig@amd.com>
0035  */
0036
0037 #include <linux/mmu_context.h>
0038 #include <linux/mempolicy.h>
0039 #include <linux/swap.h>
0040 #include <linux/sched/mm.h>
0041
0042 #include "i915_drv.h"
0043 #include "i915_gem_ioctls.h"
0044 #include "i915_gem_object.h"
0045 #include "i915_gem_userptr.h"
0046 #include "i915_scatterlist.h"
0047
0048 #ifdef CONFIG_MMU_NOTIFIER
0049
0050 /**
0051  * i915_gem_userptr_invalidate - callback to notify about mm change
0052  *
0053  * @mni: the range (mm) is about to update
0054  * @range: details on the invalidation
0055  * @cur_seq: Value to pass to mmu_interval_set_seq()
0056  *
0057  * Block for operations on BOs to finish and mark pages as accessed and
0058  * potentially dirty.
0059  */
0060 static bool i915_gem_userptr_invalidate(struct mmu_interval_notifier *mni,
0061                     const struct mmu_notifier_range *range,
0062                     unsigned long cur_seq)
0063 {
0064     struct drm_i915_gem_object *obj = container_of(mni, struct drm_i915_gem_object, userptr.notifier);
0065     struct drm_i915_private *i915 = to_i915(obj->base.dev);
0066     long r;
0067
0068     if (!mmu_notifier_range_blockable(range))
0069         return false;
0070
0071     write_lock(&i915->mm.notifier_lock);
0072
0073     mmu_interval_set_seq(mni, cur_seq);
0074
0075     write_unlock(&i915->mm.notifier_lock);
0076
0077     /*
0078      * We don't wait when the process is exiting. This is valid
0079      * because the object will be cleaned up anyway.
0080      *
0081      * This is also temporarily required as a hack, because we
0082      * cannot currently force non-consistent batch buffers to preempt
0083      * and reschedule by waiting on it, hanging processes on exit.
0084      */
0085     if (current->flags & PF_EXITING)
0086         return true;
0087
0088     /* we will unbind on next submission, still have userptr pins */
0089     r = dma_resv_wait_timeout(obj->base.resv, DMA_RESV_USAGE_BOOKKEEP, false,
0090                   MAX_SCHEDULE_TIMEOUT);
0091     if (r <= 0)
0092         drm_err(&i915->drm, "(%ld) failed to wait for idle\n", r);
0093
0094     return true;
0095 }
0096
0097 static const struct mmu_interval_notifier_ops i915_gem_userptr_notifier_ops = {
0098     .invalidate = i915_gem_userptr_invalidate,
0099 };
0100
0101 static int
0102 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj)
0103 {
0104     return mmu_interval_notifier_insert(&obj->userptr.notifier, current->mm,
0105                         obj->userptr.ptr, obj->base.size,
0106                         &i915_gem_userptr_notifier_ops);
0107 }
0108
0109 static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj)
0110 {
0111     struct page **pvec = NULL;
0112
0113     assert_object_held_shared(obj);
0114
0115     if (!--obj->userptr.page_ref) {
0116         pvec = obj->userptr.pvec;
0117         obj->userptr.pvec = NULL;
0118     }
0119     GEM_BUG_ON(obj->userptr.page_ref < 0);
0120
0121     if (pvec) {
0122         const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
0123
0124         unpin_user_pages(pvec, num_pages);
0125         kvfree(pvec);
0126     }
0127 }
0128
0129 static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
0130 {
0131     const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
0132     unsigned int max_segment = i915_sg_segment_size();
0133     struct sg_table *st;
0134     unsigned int sg_page_sizes;
0135     struct page **pvec;
0136     int ret;
0137
0138     st = kmalloc(sizeof(*st), GFP_KERNEL);
0139     if (!st)
0140         return -ENOMEM;
0141
0142     if (!obj->userptr.page_ref) {
0143         ret = -EAGAIN;
0144         goto err_free;
0145     }
0146
0147     obj->userptr.page_ref++;
0148     pvec = obj->userptr.pvec;
0149
0150 alloc_table:
0151     ret = sg_alloc_table_from_pages_segment(st, pvec, num_pages, 0,
0152                         num_pages << PAGE_SHIFT,
0153                         max_segment, GFP_KERNEL);
0154     if (ret)
0155         goto err;
0156
0157     ret = i915_gem_gtt_prepare_pages(obj, st);
0158     if (ret) {
0159         sg_free_table(st);
0160
0161         if (max_segment > PAGE_SIZE) {
0162             max_segment = PAGE_SIZE;
0163             goto alloc_table;
0164         }
0165
0166         goto err;
0167     }
0168
0169     WARN_ON_ONCE(!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE));
0170     if (i915_gem_object_can_bypass_llc(obj))
0171         obj->cache_dirty = true;
0172
0173     sg_page_sizes = i915_sg_dma_sizes(st->sgl);
0174     __i915_gem_object_set_pages(obj, st, sg_page_sizes);
0175
0176     return 0;
0177
0178 err:
0179     i915_gem_object_userptr_drop_ref(obj);
0180 err_free:
0181     kfree(st);
0182     return ret;
0183 }
0184
0185 static void
0186 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
0187                struct sg_table *pages)
0188 {
0189     struct sgt_iter sgt_iter;
0190     struct page *page;
0191
0192     if (!pages)
0193         return;
0194
0195     __i915_gem_object_release_shmem(obj, pages, true);
0196     i915_gem_gtt_finish_pages(obj, pages);
0197
0198     /*
0199      * We always mark objects as dirty when they are used by the GPU,
0200      * just in case. However, if we set the vma as being read-only we know
0201      * that the object will never have been written to.
0202      */
0203     if (i915_gem_object_is_readonly(obj))
0204         obj->mm.dirty = false;
0205
0206     for_each_sgt_page(page, sgt_iter, pages) {
0207         if (obj->mm.dirty && trylock_page(page)) {
0208             /*
0209              * As this may not be anonymous memory (e.g. shmem)
0210              * but exist on a real mapping, we have to lock
0211              * the page in order to dirty it -- holding
0212              * the page reference is not sufficient to
0213              * prevent the inode from being truncated.
0214              * Play safe and take the lock.
0215              *
0216              * However...!
0217              *
0218              * The mmu-notifier can be invalidated for a
0219              * migrate_folio, that is alreadying holding the lock
0220              * on the folio. Such a try_to_unmap() will result
0221              * in us calling put_pages() and so recursively try
0222              * to lock the page. We avoid that deadlock with
0223              * a trylock_page() and in exchange we risk missing
0224              * some page dirtying.
0225              */
0226             set_page_dirty(page);
0227             unlock_page(page);
0228         }
0229
0230         mark_page_accessed(page);
0231     }
0232     obj->mm.dirty = false;
0233
0234     sg_free_table(pages);
0235     kfree(pages);
0236
0237     i915_gem_object_userptr_drop_ref(obj);
0238 }
0239
0240 static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj)
0241 {
0242     struct sg_table *pages;
0243     int err;
0244
0245     err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
0246     if (err)
0247         return err;
0248
0249     if (GEM_WARN_ON(i915_gem_object_has_pinned_pages(obj)))
0250         return -EBUSY;
0251
0252     assert_object_held(obj);
0253
0254     pages = __i915_gem_object_unset_pages(obj);
0255     if (!IS_ERR_OR_NULL(pages))
0256         i915_gem_userptr_put_pages(obj, pages);
0257
0258     return err;
0259 }
0260
0261 int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj)
0262 {
0263     const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
0264     struct page **pvec;
0265     unsigned int gup_flags = 0;
0266     unsigned long notifier_seq;
0267     int pinned, ret;
0268
0269     if (obj->userptr.notifier.mm != current->mm)
0270         return -EFAULT;
0271
0272     notifier_seq = mmu_interval_read_begin(&obj->userptr.notifier);
0273
0274     ret = i915_gem_object_lock_interruptible(obj, NULL);
0275     if (ret)
0276         return ret;
0277
0278     if (notifier_seq == obj->userptr.notifier_seq && obj->userptr.pvec) {
0279         i915_gem_object_unlock(obj);
0280         return 0;
0281     }
0282
0283     ret = i915_gem_object_userptr_unbind(obj);
0284     i915_gem_object_unlock(obj);
0285     if (ret)
0286         return ret;
0287
0288     pvec = kvmalloc_array(num_pages, sizeof(struct page *), GFP_KERNEL);
0289     if (!pvec)
0290         return -ENOMEM;
0291
0292     if (!i915_gem_object_is_readonly(obj))
0293         gup_flags |= FOLL_WRITE;
0294
0295     pinned = ret = 0;
0296     while (pinned < num_pages) {
0297         ret = pin_user_pages_fast(obj->userptr.ptr + pinned * PAGE_SIZE,
0298                       num_pages - pinned, gup_flags,
0299                       &pvec[pinned]);
0300         if (ret < 0)
0301             goto out;
0302
0303         pinned += ret;
0304     }
0305     ret = 0;
0306
0307     ret = i915_gem_object_lock_interruptible(obj, NULL);
0308     if (ret)
0309         goto out;
0310
0311     if (mmu_interval_read_retry(&obj->userptr.notifier,
0312         !obj->userptr.page_ref ? notifier_seq :
0313         obj->userptr.notifier_seq)) {
0314         ret = -EAGAIN;
0315         goto out_unlock;
0316     }
0317
0318     if (!obj->userptr.page_ref++) {
0319         obj->userptr.pvec = pvec;
0320         obj->userptr.notifier_seq = notifier_seq;
0321         pvec = NULL;
0322         ret = ____i915_gem_object_get_pages(obj);
0323     }
0324
0325     obj->userptr.page_ref--;
0326
0327 out_unlock:
0328     i915_gem_object_unlock(obj);
0329
0330 out:
0331     if (pvec) {
0332         unpin_user_pages(pvec, pinned);
0333         kvfree(pvec);
0334     }
0335
0336     return ret;
0337 }
0338
0339 int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj)
0340 {
0341     if (mmu_interval_read_retry(&obj->userptr.notifier,
0342                     obj->userptr.notifier_seq)) {
0343         /* We collided with the mmu notifier, need to retry */
0344
0345         return -EAGAIN;
0346     }
0347
0348     return 0;
0349 }
0350
0351 int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj)
0352 {
0353     int err;
0354
0355     err = i915_gem_object_userptr_submit_init(obj);
0356     if (err)
0357         return err;
0358
0359     err = i915_gem_object_lock_interruptible(obj, NULL);
0360     if (!err) {
0361         /*
0362          * Since we only check validity, not use the pages,
0363          * it doesn't matter if we collide with the mmu notifier,
0364          * and -EAGAIN handling is not required.
0365          */
0366         err = i915_gem_object_pin_pages(obj);
0367         if (!err)
0368             i915_gem_object_unpin_pages(obj);
0369
0370         i915_gem_object_unlock(obj);
0371     }
0372
0373     return err;
0374 }
0375
0376 static void
0377 i915_gem_userptr_release(struct drm_i915_gem_object *obj)
0378 {
0379     GEM_WARN_ON(obj->userptr.page_ref);
0380
0381     mmu_interval_notifier_remove(&obj->userptr.notifier);
0382     obj->userptr.notifier.mm = NULL;
0383 }
0384
0385 static int
0386 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
0387 {
0388     drm_dbg(obj->base.dev, "Exporting userptr no longer allowed\n");
0389
0390     return -EINVAL;
0391 }
0392
0393 static int
0394 i915_gem_userptr_pwrite(struct drm_i915_gem_object *obj,
0395             const struct drm_i915_gem_pwrite *args)
0396 {
0397     drm_dbg(obj->base.dev, "pwrite to userptr no longer allowed\n");
0398
0399     return -EINVAL;
0400 }
0401
0402 static int
0403 i915_gem_userptr_pread(struct drm_i915_gem_object *obj,
0404                const struct drm_i915_gem_pread *args)
0405 {
0406     drm_dbg(obj->base.dev, "pread from userptr no longer allowed\n");
0407
0408     return -EINVAL;
0409 }
0410
0411 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
0412     .name = "i915_gem_object_userptr",
0413     .flags = I915_GEM_OBJECT_IS_SHRINKABLE |
0414          I915_GEM_OBJECT_NO_MMAP |
0415          I915_GEM_OBJECT_IS_PROXY,
0416     .get_pages = i915_gem_userptr_get_pages,
0417     .put_pages = i915_gem_userptr_put_pages,
0418     .dmabuf_export = i915_gem_userptr_dmabuf_export,
0419     .pwrite = i915_gem_userptr_pwrite,
0420     .pread = i915_gem_userptr_pread,
0421     .release = i915_gem_userptr_release,
0422 };
0423
0424 #endif
0425
0426 static int
0427 probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len)
0428 {
0429     const unsigned long end = addr + len;
0430     struct vm_area_struct *vma;
0431     int ret = -EFAULT;
0432
0433     mmap_read_lock(mm);
0434     for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
0435         /* Check for holes, note that we also update the addr below */
0436         if (vma->vm_start > addr)
0437             break;
0438
0439         if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
0440             break;
0441
0442         if (vma->vm_end >= end) {
0443             ret = 0;
0444             break;
0445         }
0446
0447         addr = vma->vm_end;
0448     }
0449     mmap_read_unlock(mm);
0450
0451     return ret;
0452 }
0453
0454 /*
0455  * Creates a new mm object that wraps some normal memory from the process
0456  * context - user memory.
0457  *
0458  * We impose several restrictions upon the memory being mapped
0459  * into the GPU.
0460  * 1. It must be page aligned (both start/end addresses, i.e ptr and size).
0461  * 2. It must be normal system memory, not a pointer into another map of IO
0462  *    space (e.g. it must not be a GTT mmapping of another object).
0463  * 3. We only allow a bo as large as we could in theory map into the GTT,
0464  *    that is we limit the size to the total size of the GTT.
0465  * 4. The bo is marked as being snoopable. The backing pages are left
0466  *    accessible directly by the CPU, but reads and writes by the GPU may
0467  *    incur the cost of a snoop (unless you have an LLC architecture).
0468  *
0469  * Synchronisation between multiple users and the GPU is left to userspace
0470  * through the normal set-domain-ioctl. The kernel will enforce that the
0471  * GPU relinquishes the VMA before it is returned back to the system
0472  * i.e. upon free(), munmap() or process termination. However, the userspace
0473  * malloc() library may not immediately relinquish the VMA after free() and
0474  * instead reuse it whilst the GPU is still reading and writing to the VMA.
0475  * Caveat emptor.
0476  *
0477  * Also note, that the object created here is not currently a "first class"
0478  * object, in that several ioctls are banned. These are the CPU access
0479  * ioctls: mmap(), pwrite and pread. In practice, you are expected to use
0480  * direct access via your pointer rather than use those ioctls. Another
0481  * restriction is that we do not allow userptr surfaces to be pinned to the
0482  * hardware and so we reject any attempt to create a framebuffer out of a
0483  * userptr.
0484  *
0485  * If you think this is a good interface to use to pass GPU memory between
0486  * drivers, please use dma-buf instead. In fact, wherever possible use
0487  * dma-buf instead.
0488  */
0489 int
0490 i915_gem_userptr_ioctl(struct drm_device *dev,
0491                void *data,
0492                struct drm_file *file)
0493 {
0494     static struct lock_class_key __maybe_unused lock_class;
0495     struct drm_i915_private *dev_priv = to_i915(dev);
0496     struct drm_i915_gem_userptr *args = data;
0497     struct drm_i915_gem_object __maybe_unused *obj;
0498     int __maybe_unused ret;
0499     u32 __maybe_unused handle;
0500
0501     if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
0502         /* We cannot support coherent userptr objects on hw without
0503          * LLC and broken snooping.
0504          */
0505         return -ENODEV;
0506     }
0507
0508     if (args->flags & ~(I915_USERPTR_READ_ONLY |
0509                 I915_USERPTR_UNSYNCHRONIZED |
0510                 I915_USERPTR_PROBE))
0511         return -EINVAL;
0512
0513     if (i915_gem_object_size_2big(args->user_size))
0514         return -E2BIG;
0515
0516     if (!args->user_size)
0517         return -EINVAL;
0518
0519     if (offset_in_page(args->user_ptr | args->user_size))
0520         return -EINVAL;
0521
0522     if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size))
0523         return -EFAULT;
0524
0525     if (args->flags & I915_USERPTR_UNSYNCHRONIZED)
0526         return -ENODEV;
0527
0528     if (args->flags & I915_USERPTR_READ_ONLY) {
0529         /*
0530          * On almost all of the older hw, we cannot tell the GPU that
0531          * a page is readonly.
0532          */
0533         if (!to_gt(dev_priv)->vm->has_read_only)
0534             return -ENODEV;
0535     }
0536
0537     if (args->flags & I915_USERPTR_PROBE) {
0538         /*
0539          * Check that the range pointed to represents real struct
0540          * pages and not iomappings (at this moment in time!)
0541          */
0542         ret = probe_range(current->mm, args->user_ptr, args->user_size);
0543         if (ret)
0544             return ret;
0545     }
0546
0547 #ifdef CONFIG_MMU_NOTIFIER
0548     obj = i915_gem_object_alloc();
0549     if (obj == NULL)
0550         return -ENOMEM;
0551
0552     drm_gem_private_object_init(dev, &obj->base, args->user_size);
0553     i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class,
0554                  I915_BO_ALLOC_USER);
0555     obj->mem_flags = I915_BO_FLAG_STRUCT_PAGE;
0556     obj->read_domains = I915_GEM_DOMAIN_CPU;
0557     obj->write_domain = I915_GEM_DOMAIN_CPU;
0558     i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
0559
0560     obj->userptr.ptr = args->user_ptr;
0561     obj->userptr.notifier_seq = ULONG_MAX;
0562     if (args->flags & I915_USERPTR_READ_ONLY)
0563         i915_gem_object_set_readonly(obj);
0564
0565     /* And keep a pointer to the current->mm for resolving the user pages
0566      * at binding. This means that we need to hook into the mmu_notifier
0567      * in order to detect if the mmu is destroyed.
0568      */
0569     ret = i915_gem_userptr_init__mmu_notifier(obj);
0570     if (ret == 0)
0571         ret = drm_gem_handle_create(file, &obj->base, &handle);
0572
0573     /* drop reference from allocate - handle holds it now */
0574     i915_gem_object_put(obj);
0575     if (ret)
0576         return ret;
0577
0578     args->handle = handle;
0579     return 0;
0580 #else
0581     return -ENODEV;
0582 #endif
0583 }
0584
0585 int i915_gem_init_userptr(struct drm_i915_private *dev_priv)
0586 {
0587 #ifdef CONFIG_MMU_NOTIFIER
0588     rwlock_init(&dev_priv->mm.notifier_lock);
0589 #endif
0590
0591     return 0;
0592 }
0593
0594 void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)
0595 {
0596 }