Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2021 Intel Corporation
0004  */
0005 
0006 #include <drm/ttm/ttm_bo_driver.h>
0007 
0008 #include "i915_deps.h"
0009 #include "i915_drv.h"
0010 #include "intel_memory_region.h"
0011 #include "intel_region_ttm.h"
0012 
0013 #include "gem/i915_gem_object.h"
0014 #include "gem/i915_gem_region.h"
0015 #include "gem/i915_gem_ttm.h"
0016 #include "gem/i915_gem_ttm_move.h"
0017 
0018 #include "gt/intel_engine_pm.h"
0019 #include "gt/intel_gt.h"
0020 #include "gt/intel_migrate.h"
0021 
0022 /**
0023  * DOC: Selftest failure modes for failsafe migration:
0024  *
0025  * For fail_gpu_migration, the gpu blit scheduled is always a clear blit
0026  * rather than a copy blit, and then we force the failure paths as if
0027  * the blit fence returned an error.
0028  *
0029  * For fail_work_allocation we fail the kmalloc of the async worker, we
0030  * sync the gpu blit. If it then fails, or fail_gpu_migration is set to
0031  * true, then a memcpy operation is performed sync.
0032  */
0033 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
0034 static bool fail_gpu_migration;
0035 static bool fail_work_allocation;
0036 static bool ban_memcpy;
0037 
0038 void i915_ttm_migrate_set_failure_modes(bool gpu_migration,
0039                     bool work_allocation)
0040 {
0041     fail_gpu_migration = gpu_migration;
0042     fail_work_allocation = work_allocation;
0043 }
0044 
0045 void i915_ttm_migrate_set_ban_memcpy(bool ban)
0046 {
0047     ban_memcpy = ban;
0048 }
0049 #endif
0050 
0051 static enum i915_cache_level
0052 i915_ttm_cache_level(struct drm_i915_private *i915, struct ttm_resource *res,
0053              struct ttm_tt *ttm)
0054 {
0055     return ((HAS_LLC(i915) || HAS_SNOOP(i915)) &&
0056         !i915_ttm_gtt_binds_lmem(res) &&
0057         ttm->caching == ttm_cached) ? I915_CACHE_LLC :
0058         I915_CACHE_NONE;
0059 }
0060 
0061 static struct intel_memory_region *
0062 i915_ttm_region(struct ttm_device *bdev, int ttm_mem_type)
0063 {
0064     struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev);
0065 
0066     /* There's some room for optimization here... */
0067     GEM_BUG_ON(ttm_mem_type != I915_PL_SYSTEM &&
0068            ttm_mem_type < I915_PL_LMEM0);
0069     if (ttm_mem_type == I915_PL_SYSTEM)
0070         return intel_memory_region_lookup(i915, INTEL_MEMORY_SYSTEM,
0071                           0);
0072 
0073     return intel_memory_region_lookup(i915, INTEL_MEMORY_LOCAL,
0074                       ttm_mem_type - I915_PL_LMEM0);
0075 }
0076 
0077 /**
0078  * i915_ttm_adjust_domains_after_move - Adjust the GEM domains after a
0079  * TTM move
0080  * @obj: The gem object
0081  */
0082 void i915_ttm_adjust_domains_after_move(struct drm_i915_gem_object *obj)
0083 {
0084     struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
0085 
0086     if (i915_ttm_cpu_maps_iomem(bo->resource) || bo->ttm->caching != ttm_cached) {
0087         obj->write_domain = I915_GEM_DOMAIN_WC;
0088         obj->read_domains = I915_GEM_DOMAIN_WC;
0089     } else {
0090         obj->write_domain = I915_GEM_DOMAIN_CPU;
0091         obj->read_domains = I915_GEM_DOMAIN_CPU;
0092     }
0093 }
0094 
0095 /**
0096  * i915_ttm_adjust_gem_after_move - Adjust the GEM state after a TTM move
0097  * @obj: The gem object
0098  *
0099  * Adjusts the GEM object's region, mem_flags and cache coherency after a
0100  * TTM move.
0101  */
0102 void i915_ttm_adjust_gem_after_move(struct drm_i915_gem_object *obj)
0103 {
0104     struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
0105     unsigned int cache_level;
0106     unsigned int i;
0107 
0108     /*
0109      * If object was moved to an allowable region, update the object
0110      * region to consider it migrated. Note that if it's currently not
0111      * in an allowable region, it's evicted and we don't update the
0112      * object region.
0113      */
0114     if (intel_region_to_ttm_type(obj->mm.region) != bo->resource->mem_type) {
0115         for (i = 0; i < obj->mm.n_placements; ++i) {
0116             struct intel_memory_region *mr = obj->mm.placements[i];
0117 
0118             if (intel_region_to_ttm_type(mr) == bo->resource->mem_type &&
0119                 mr != obj->mm.region) {
0120                 i915_gem_object_release_memory_region(obj);
0121                 i915_gem_object_init_memory_region(obj, mr);
0122                 break;
0123             }
0124         }
0125     }
0126 
0127     obj->mem_flags &= ~(I915_BO_FLAG_STRUCT_PAGE | I915_BO_FLAG_IOMEM);
0128 
0129     obj->mem_flags |= i915_ttm_cpu_maps_iomem(bo->resource) ? I915_BO_FLAG_IOMEM :
0130         I915_BO_FLAG_STRUCT_PAGE;
0131 
0132     cache_level = i915_ttm_cache_level(to_i915(bo->base.dev), bo->resource,
0133                        bo->ttm);
0134     i915_gem_object_set_cache_coherency(obj, cache_level);
0135 }
0136 
0137 /**
0138  * i915_ttm_move_notify - Prepare an object for move
0139  * @bo: The ttm buffer object.
0140  *
0141  * This function prepares an object for move by removing all GPU bindings,
0142  * removing all CPU mapings and finally releasing the pages sg-table.
0143  *
0144  * Return: 0 if successful, negative error code on error.
0145  */
0146 int i915_ttm_move_notify(struct ttm_buffer_object *bo)
0147 {
0148     struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
0149     int ret;
0150 
0151     /*
0152      * Note: The async unbinding here will actually transform the
0153      * blocking wait for unbind into a wait before finally submitting
0154      * evict / migration blit and thus stall the migration timeline
0155      * which may not be good for overall throughput. We should make
0156      * sure we await the unbind fences *after* the migration blit
0157      * instead of *before* as we currently do.
0158      */
0159     ret = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE |
0160                      I915_GEM_OBJECT_UNBIND_ASYNC);
0161     if (ret)
0162         return ret;
0163 
0164     ret = __i915_gem_object_put_pages(obj);
0165     if (ret)
0166         return ret;
0167 
0168     return 0;
0169 }
0170 
0171 static struct dma_fence *i915_ttm_accel_move(struct ttm_buffer_object *bo,
0172                          bool clear,
0173                          struct ttm_resource *dst_mem,
0174                          struct ttm_tt *dst_ttm,
0175                          struct sg_table *dst_st,
0176                          const struct i915_deps *deps)
0177 {
0178     struct drm_i915_private *i915 = container_of(bo->bdev, typeof(*i915),
0179                              bdev);
0180     struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
0181     struct i915_request *rq;
0182     struct ttm_tt *src_ttm = bo->ttm;
0183     enum i915_cache_level src_level, dst_level;
0184     int ret;
0185 
0186     if (!to_gt(i915)->migrate.context || intel_gt_is_wedged(to_gt(i915)))
0187         return ERR_PTR(-EINVAL);
0188 
0189     /* With fail_gpu_migration, we always perform a GPU clear. */
0190     if (I915_SELFTEST_ONLY(fail_gpu_migration))
0191         clear = true;
0192 
0193     dst_level = i915_ttm_cache_level(i915, dst_mem, dst_ttm);
0194     if (clear) {
0195         if (bo->type == ttm_bo_type_kernel &&
0196             !I915_SELFTEST_ONLY(fail_gpu_migration))
0197             return ERR_PTR(-EINVAL);
0198 
0199         intel_engine_pm_get(to_gt(i915)->migrate.context->engine);
0200         ret = intel_context_migrate_clear(to_gt(i915)->migrate.context, deps,
0201                           dst_st->sgl, dst_level,
0202                           i915_ttm_gtt_binds_lmem(dst_mem),
0203                           0, &rq);
0204     } else {
0205         struct i915_refct_sgt *src_rsgt =
0206             i915_ttm_resource_get_st(obj, bo->resource);
0207 
0208         if (IS_ERR(src_rsgt))
0209             return ERR_CAST(src_rsgt);
0210 
0211         src_level = i915_ttm_cache_level(i915, bo->resource, src_ttm);
0212         intel_engine_pm_get(to_gt(i915)->migrate.context->engine);
0213         ret = intel_context_migrate_copy(to_gt(i915)->migrate.context,
0214                          deps, src_rsgt->table.sgl,
0215                          src_level,
0216                          i915_ttm_gtt_binds_lmem(bo->resource),
0217                          dst_st->sgl, dst_level,
0218                          i915_ttm_gtt_binds_lmem(dst_mem),
0219                          &rq);
0220 
0221         i915_refct_sgt_put(src_rsgt);
0222     }
0223 
0224     intel_engine_pm_put(to_gt(i915)->migrate.context->engine);
0225 
0226     if (ret && rq) {
0227         i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
0228         i915_request_put(rq);
0229     }
0230 
0231     return ret ? ERR_PTR(ret) : &rq->fence;
0232 }
0233 
0234 /**
0235  * struct i915_ttm_memcpy_arg - argument for the bo memcpy functionality.
0236  * @_dst_iter: Storage space for the destination kmap iterator.
0237  * @_src_iter: Storage space for the source kmap iterator.
0238  * @dst_iter: Pointer to the destination kmap iterator.
0239  * @src_iter: Pointer to the source kmap iterator.
0240  * @clear: Whether to clear instead of copy.
0241  * @src_rsgt: Refcounted scatter-gather list of source memory.
0242  * @dst_rsgt: Refcounted scatter-gather list of destination memory.
0243  */
0244 struct i915_ttm_memcpy_arg {
0245     union {
0246         struct ttm_kmap_iter_tt tt;
0247         struct ttm_kmap_iter_iomap io;
0248     } _dst_iter,
0249     _src_iter;
0250     struct ttm_kmap_iter *dst_iter;
0251     struct ttm_kmap_iter *src_iter;
0252     unsigned long num_pages;
0253     bool clear;
0254     struct i915_refct_sgt *src_rsgt;
0255     struct i915_refct_sgt *dst_rsgt;
0256 };
0257 
0258 /**
0259  * struct i915_ttm_memcpy_work - Async memcpy worker under a dma-fence.
0260  * @fence: The dma-fence.
0261  * @work: The work struct use for the memcpy work.
0262  * @lock: The fence lock. Not used to protect anything else ATM.
0263  * @irq_work: Low latency worker to signal the fence since it can't be done
0264  * from the callback for lockdep reasons.
0265  * @cb: Callback for the accelerated migration fence.
0266  * @arg: The argument for the memcpy functionality.
0267  * @i915: The i915 pointer.
0268  * @obj: The GEM object.
0269  * @memcpy_allowed: Instead of processing the @arg, and falling back to memcpy
0270  * or memset, we wedge the device and set the @obj unknown_state, to prevent
0271  * further access to the object with the CPU or GPU.  On some devices we might
0272  * only be permitted to use the blitter engine for such operations.
0273  */
0274 struct i915_ttm_memcpy_work {
0275     struct dma_fence fence;
0276     struct work_struct work;
0277     spinlock_t lock;
0278     struct irq_work irq_work;
0279     struct dma_fence_cb cb;
0280     struct i915_ttm_memcpy_arg arg;
0281     struct drm_i915_private *i915;
0282     struct drm_i915_gem_object *obj;
0283     bool memcpy_allowed;
0284 };
0285 
0286 static void i915_ttm_move_memcpy(struct i915_ttm_memcpy_arg *arg)
0287 {
0288     ttm_move_memcpy(arg->clear, arg->num_pages,
0289             arg->dst_iter, arg->src_iter);
0290 }
0291 
0292 static void i915_ttm_memcpy_init(struct i915_ttm_memcpy_arg *arg,
0293                  struct ttm_buffer_object *bo, bool clear,
0294                  struct ttm_resource *dst_mem,
0295                  struct ttm_tt *dst_ttm,
0296                  struct i915_refct_sgt *dst_rsgt)
0297 {
0298     struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
0299     struct intel_memory_region *dst_reg, *src_reg;
0300 
0301     dst_reg = i915_ttm_region(bo->bdev, dst_mem->mem_type);
0302     src_reg = i915_ttm_region(bo->bdev, bo->resource->mem_type);
0303     GEM_BUG_ON(!dst_reg || !src_reg);
0304 
0305     arg->dst_iter = !i915_ttm_cpu_maps_iomem(dst_mem) ?
0306         ttm_kmap_iter_tt_init(&arg->_dst_iter.tt, dst_ttm) :
0307         ttm_kmap_iter_iomap_init(&arg->_dst_iter.io, &dst_reg->iomap,
0308                      &dst_rsgt->table, dst_reg->region.start);
0309 
0310     arg->src_iter = !i915_ttm_cpu_maps_iomem(bo->resource) ?
0311         ttm_kmap_iter_tt_init(&arg->_src_iter.tt, bo->ttm) :
0312         ttm_kmap_iter_iomap_init(&arg->_src_iter.io, &src_reg->iomap,
0313                      &obj->ttm.cached_io_rsgt->table,
0314                      src_reg->region.start);
0315     arg->clear = clear;
0316     arg->num_pages = bo->base.size >> PAGE_SHIFT;
0317 
0318     arg->dst_rsgt = i915_refct_sgt_get(dst_rsgt);
0319     arg->src_rsgt = clear ? NULL :
0320         i915_ttm_resource_get_st(obj, bo->resource);
0321 }
0322 
0323 static void i915_ttm_memcpy_release(struct i915_ttm_memcpy_arg *arg)
0324 {
0325     i915_refct_sgt_put(arg->src_rsgt);
0326     i915_refct_sgt_put(arg->dst_rsgt);
0327 }
0328 
0329 static void __memcpy_work(struct work_struct *work)
0330 {
0331     struct i915_ttm_memcpy_work *copy_work =
0332         container_of(work, typeof(*copy_work), work);
0333     struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
0334     bool cookie;
0335 
0336     /*
0337      * FIXME: We need to take a closer look here. We should be able to plonk
0338      * this into the fence critical section.
0339      */
0340     if (!copy_work->memcpy_allowed) {
0341         struct intel_gt *gt;
0342         unsigned int id;
0343 
0344         for_each_gt(gt, copy_work->i915, id)
0345             intel_gt_set_wedged(gt);
0346     }
0347 
0348     cookie = dma_fence_begin_signalling();
0349 
0350     if (copy_work->memcpy_allowed) {
0351         i915_ttm_move_memcpy(arg);
0352     } else {
0353         /*
0354          * Prevent further use of the object. Any future GTT binding or
0355          * CPU access is not allowed once we signal the fence. Outside
0356          * of the fence critical section, we then also then wedge the gpu
0357          * to indicate the device is not functional.
0358          *
0359          * The below dma_fence_signal() is our write-memory-barrier.
0360          */
0361         copy_work->obj->mm.unknown_state = true;
0362     }
0363 
0364     dma_fence_end_signalling(cookie);
0365 
0366     dma_fence_signal(&copy_work->fence);
0367 
0368     i915_ttm_memcpy_release(arg);
0369     i915_gem_object_put(copy_work->obj);
0370     dma_fence_put(&copy_work->fence);
0371 }
0372 
0373 static void __memcpy_irq_work(struct irq_work *irq_work)
0374 {
0375     struct i915_ttm_memcpy_work *copy_work =
0376         container_of(irq_work, typeof(*copy_work), irq_work);
0377     struct i915_ttm_memcpy_arg *arg = &copy_work->arg;
0378 
0379     dma_fence_signal(&copy_work->fence);
0380     i915_ttm_memcpy_release(arg);
0381     i915_gem_object_put(copy_work->obj);
0382     dma_fence_put(&copy_work->fence);
0383 }
0384 
0385 static void __memcpy_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
0386 {
0387     struct i915_ttm_memcpy_work *copy_work =
0388         container_of(cb, typeof(*copy_work), cb);
0389 
0390     if (unlikely(fence->error || I915_SELFTEST_ONLY(fail_gpu_migration))) {
0391         INIT_WORK(&copy_work->work, __memcpy_work);
0392         queue_work(system_unbound_wq, &copy_work->work);
0393     } else {
0394         init_irq_work(&copy_work->irq_work, __memcpy_irq_work);
0395         irq_work_queue(&copy_work->irq_work);
0396     }
0397 }
0398 
0399 static const char *get_driver_name(struct dma_fence *fence)
0400 {
0401     return "i915_ttm_memcpy_work";
0402 }
0403 
0404 static const char *get_timeline_name(struct dma_fence *fence)
0405 {
0406     return "unbound";
0407 }
0408 
0409 static const struct dma_fence_ops dma_fence_memcpy_ops = {
0410     .get_driver_name = get_driver_name,
0411     .get_timeline_name = get_timeline_name,
0412 };
0413 
0414 static struct dma_fence *
0415 i915_ttm_memcpy_work_arm(struct i915_ttm_memcpy_work *work,
0416              struct dma_fence *dep)
0417 {
0418     int ret;
0419 
0420     spin_lock_init(&work->lock);
0421     dma_fence_init(&work->fence, &dma_fence_memcpy_ops, &work->lock, 0, 0);
0422     dma_fence_get(&work->fence);
0423     ret = dma_fence_add_callback(dep, &work->cb, __memcpy_cb);
0424     if (ret) {
0425         if (ret != -ENOENT)
0426             dma_fence_wait(dep, false);
0427 
0428         return ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ? -EINVAL :
0429                    dep->error);
0430     }
0431 
0432     return &work->fence;
0433 }
0434 
0435 static bool i915_ttm_memcpy_allowed(struct ttm_buffer_object *bo,
0436                     struct ttm_resource *dst_mem)
0437 {
0438     if (i915_gem_object_needs_ccs_pages(i915_ttm_to_gem(bo)))
0439         return false;
0440 
0441     if (!(i915_ttm_resource_mappable(bo->resource) &&
0442           i915_ttm_resource_mappable(dst_mem)))
0443         return false;
0444 
0445     return I915_SELFTEST_ONLY(ban_memcpy) ? false : true;
0446 }
0447 
0448 static struct dma_fence *
0449 __i915_ttm_move(struct ttm_buffer_object *bo,
0450         const struct ttm_operation_ctx *ctx, bool clear,
0451         struct ttm_resource *dst_mem, struct ttm_tt *dst_ttm,
0452         struct i915_refct_sgt *dst_rsgt, bool allow_accel,
0453         const struct i915_deps *move_deps)
0454 {
0455     const bool memcpy_allowed = i915_ttm_memcpy_allowed(bo, dst_mem);
0456     struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
0457     struct drm_i915_private *i915 = to_i915(bo->base.dev);
0458     struct i915_ttm_memcpy_work *copy_work = NULL;
0459     struct i915_ttm_memcpy_arg _arg, *arg = &_arg;
0460     struct dma_fence *fence = ERR_PTR(-EINVAL);
0461 
0462     if (allow_accel) {
0463         fence = i915_ttm_accel_move(bo, clear, dst_mem, dst_ttm,
0464                         &dst_rsgt->table, move_deps);
0465 
0466         /*
0467          * We only need to intercept the error when moving to lmem.
0468          * When moving to system, TTM or shmem will provide us with
0469          * cleared pages.
0470          */
0471         if (!IS_ERR(fence) && !i915_ttm_gtt_binds_lmem(dst_mem) &&
0472             !I915_SELFTEST_ONLY(fail_gpu_migration ||
0473                     fail_work_allocation))
0474             goto out;
0475     }
0476 
0477     /* If we've scheduled gpu migration. Try to arm error intercept. */
0478     if (!IS_ERR(fence)) {
0479         struct dma_fence *dep = fence;
0480 
0481         if (!I915_SELFTEST_ONLY(fail_work_allocation))
0482             copy_work = kzalloc(sizeof(*copy_work), GFP_KERNEL);
0483 
0484         if (copy_work) {
0485             copy_work->i915 = i915;
0486             copy_work->memcpy_allowed = memcpy_allowed;
0487             copy_work->obj = i915_gem_object_get(obj);
0488             arg = &copy_work->arg;
0489             if (memcpy_allowed)
0490                 i915_ttm_memcpy_init(arg, bo, clear, dst_mem,
0491                              dst_ttm, dst_rsgt);
0492 
0493             fence = i915_ttm_memcpy_work_arm(copy_work, dep);
0494         } else {
0495             dma_fence_wait(dep, false);
0496             fence = ERR_PTR(I915_SELFTEST_ONLY(fail_gpu_migration) ?
0497                     -EINVAL : fence->error);
0498         }
0499         dma_fence_put(dep);
0500 
0501         if (!IS_ERR(fence))
0502             goto out;
0503     } else {
0504         int err = PTR_ERR(fence);
0505 
0506         if (err == -EINTR || err == -ERESTARTSYS || err == -EAGAIN)
0507             return fence;
0508 
0509         if (move_deps) {
0510             err = i915_deps_sync(move_deps, ctx);
0511             if (err)
0512                 return ERR_PTR(err);
0513         }
0514     }
0515 
0516     /* Error intercept failed or no accelerated migration to start with */
0517 
0518     if (memcpy_allowed) {
0519         if (!copy_work)
0520             i915_ttm_memcpy_init(arg, bo, clear, dst_mem, dst_ttm,
0521                          dst_rsgt);
0522         i915_ttm_move_memcpy(arg);
0523         i915_ttm_memcpy_release(arg);
0524     }
0525     if (copy_work)
0526         i915_gem_object_put(copy_work->obj);
0527     kfree(copy_work);
0528 
0529     return memcpy_allowed ? NULL : ERR_PTR(-EIO);
0530 out:
0531     if (!fence && copy_work) {
0532         i915_ttm_memcpy_release(arg);
0533         i915_gem_object_put(copy_work->obj);
0534         kfree(copy_work);
0535     }
0536 
0537     return fence;
0538 }
0539 
0540 /**
0541  * i915_ttm_move - The TTM move callback used by i915.
0542  * @bo: The buffer object.
0543  * @evict: Whether this is an eviction.
0544  * @dst_mem: The destination ttm resource.
0545  * @hop: If we need multihop, what temporary memory type to move to.
0546  *
0547  * Return: 0 if successful, negative error code otherwise.
0548  */
0549 int i915_ttm_move(struct ttm_buffer_object *bo, bool evict,
0550           struct ttm_operation_ctx *ctx,
0551           struct ttm_resource *dst_mem,
0552           struct ttm_place *hop)
0553 {
0554     struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
0555     struct ttm_resource_manager *dst_man =
0556         ttm_manager_type(bo->bdev, dst_mem->mem_type);
0557     struct dma_fence *migration_fence = NULL;
0558     struct ttm_tt *ttm = bo->ttm;
0559     struct i915_refct_sgt *dst_rsgt;
0560     bool clear;
0561     int ret;
0562 
0563     if (GEM_WARN_ON(!obj)) {
0564         ttm_bo_move_null(bo, dst_mem);
0565         return 0;
0566     }
0567 
0568     ret = i915_ttm_move_notify(bo);
0569     if (ret)
0570         return ret;
0571 
0572     if (obj->mm.madv != I915_MADV_WILLNEED) {
0573         i915_ttm_purge(obj);
0574         ttm_resource_free(bo, &dst_mem);
0575         return 0;
0576     }
0577 
0578     /* Populate ttm with pages if needed. Typically system memory. */
0579     if (ttm && (dst_man->use_tt || (ttm->page_flags & TTM_TT_FLAG_SWAPPED))) {
0580         ret = ttm_tt_populate(bo->bdev, ttm, ctx);
0581         if (ret)
0582             return ret;
0583     }
0584 
0585     dst_rsgt = i915_ttm_resource_get_st(obj, dst_mem);
0586     if (IS_ERR(dst_rsgt))
0587         return PTR_ERR(dst_rsgt);
0588 
0589     clear = !i915_ttm_cpu_maps_iomem(bo->resource) && (!ttm || !ttm_tt_is_populated(ttm));
0590     if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) {
0591         struct i915_deps deps;
0592 
0593         i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
0594         ret = i915_deps_add_resv(&deps, bo->base.resv, ctx);
0595         if (ret) {
0596             i915_refct_sgt_put(dst_rsgt);
0597             return ret;
0598         }
0599 
0600         migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, ttm,
0601                           dst_rsgt, true, &deps);
0602         i915_deps_fini(&deps);
0603     }
0604 
0605     /* We can possibly get an -ERESTARTSYS here */
0606     if (IS_ERR(migration_fence)) {
0607         i915_refct_sgt_put(dst_rsgt);
0608         return PTR_ERR(migration_fence);
0609     }
0610 
0611     if (migration_fence) {
0612         if (I915_SELFTEST_ONLY(evict && fail_gpu_migration))
0613             ret = -EIO; /* never feed non-migrate fences into ttm */
0614         else
0615             ret = ttm_bo_move_accel_cleanup(bo, migration_fence, evict,
0616                             true, dst_mem);
0617         if (ret) {
0618             dma_fence_wait(migration_fence, false);
0619             ttm_bo_move_sync_cleanup(bo, dst_mem);
0620         }
0621         dma_fence_put(migration_fence);
0622     } else {
0623         ttm_bo_move_sync_cleanup(bo, dst_mem);
0624     }
0625 
0626     i915_ttm_adjust_domains_after_move(obj);
0627     i915_ttm_free_cached_io_rsgt(obj);
0628 
0629     if (i915_ttm_gtt_binds_lmem(dst_mem) || i915_ttm_cpu_maps_iomem(dst_mem)) {
0630         obj->ttm.cached_io_rsgt = dst_rsgt;
0631         obj->ttm.get_io_page.sg_pos = dst_rsgt->table.sgl;
0632         obj->ttm.get_io_page.sg_idx = 0;
0633     } else {
0634         i915_refct_sgt_put(dst_rsgt);
0635     }
0636 
0637     i915_ttm_adjust_lru(obj);
0638     i915_ttm_adjust_gem_after_move(obj);
0639     return 0;
0640 }
0641 
0642 /**
0643  * i915_gem_obj_copy_ttm - Copy the contents of one ttm-based gem object to
0644  * another
0645  * @dst: The destination object
0646  * @src: The source object
0647  * @allow_accel: Allow using the blitter. Otherwise TTM memcpy is used.
0648  * @intr: Whether to perform waits interruptible:
0649  *
0650  * Note: The caller is responsible for assuring that the underlying
0651  * TTM objects are populated if needed and locked.
0652  *
0653  * Return: Zero on success. Negative error code on error. If @intr == true,
0654  * then it may return -ERESTARTSYS or -EINTR.
0655  */
0656 int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst,
0657               struct drm_i915_gem_object *src,
0658               bool allow_accel, bool intr)
0659 {
0660     struct ttm_buffer_object *dst_bo = i915_gem_to_ttm(dst);
0661     struct ttm_buffer_object *src_bo = i915_gem_to_ttm(src);
0662     struct ttm_operation_ctx ctx = {
0663         .interruptible = intr,
0664     };
0665     struct i915_refct_sgt *dst_rsgt;
0666     struct dma_fence *copy_fence;
0667     struct i915_deps deps;
0668     int ret;
0669 
0670     assert_object_held(dst);
0671     assert_object_held(src);
0672     i915_deps_init(&deps, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
0673 
0674     ret = dma_resv_reserve_fences(src_bo->base.resv, 1);
0675     if (ret)
0676         return ret;
0677 
0678     ret = dma_resv_reserve_fences(dst_bo->base.resv, 1);
0679     if (ret)
0680         return ret;
0681 
0682     ret = i915_deps_add_resv(&deps, dst_bo->base.resv, &ctx);
0683     if (ret)
0684         return ret;
0685 
0686     ret = i915_deps_add_resv(&deps, src_bo->base.resv, &ctx);
0687     if (ret)
0688         return ret;
0689 
0690     dst_rsgt = i915_ttm_resource_get_st(dst, dst_bo->resource);
0691     copy_fence = __i915_ttm_move(src_bo, &ctx, false, dst_bo->resource,
0692                      dst_bo->ttm, dst_rsgt, allow_accel,
0693                      &deps);
0694 
0695     i915_deps_fini(&deps);
0696     i915_refct_sgt_put(dst_rsgt);
0697     if (IS_ERR_OR_NULL(copy_fence))
0698         return PTR_ERR_OR_ZERO(copy_fence);
0699 
0700     dma_resv_add_fence(dst_bo->base.resv, copy_fence, DMA_RESV_USAGE_WRITE);
0701     dma_resv_add_fence(src_bo->base.resv, copy_fence, DMA_RESV_USAGE_READ);
0702     dma_fence_put(copy_fence);
0703 
0704     return 0;
0705 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!