amd/amdgpu/amdgpu_amdkfd_fence.c

0001 /*
0002  * Copyright 2016-2018 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  */
0022
0023 #include <linux/dma-fence.h>
0024 #include <linux/spinlock.h>
0025 #include <linux/atomic.h>
0026 #include <linux/stacktrace.h>
0027 #include <linux/sched.h>
0028 #include <linux/slab.h>
0029 #include <linux/sched/mm.h>
0030 #include "amdgpu_amdkfd.h"
0031 #include "kfd_svm.h"
0032
0033 static const struct dma_fence_ops amdkfd_fence_ops;
0034 static atomic_t fence_seq = ATOMIC_INIT(0);
0035
0036 /* Eviction Fence
0037  * Fence helper functions to deal with KFD memory eviction.
0038  * Big Idea - Since KFD submissions are done by user queues, a BO cannot be
0039  *  evicted unless all the user queues for that process are evicted.
0040  *
0041  * All the BOs in a process share an eviction fence. When process X wants
0042  * to map VRAM memory but TTM can't find enough space, TTM will attempt to
0043  * evict BOs from its LRU list. TTM checks if the BO is valuable to evict
0044  * by calling ttm_device_funcs->eviction_valuable().
0045  *
0046  * ttm_device_funcs->eviction_valuable() - will return false if the BO belongs
0047  *  to process X. Otherwise, it will return true to indicate BO can be
0048  *  evicted by TTM.
0049  *
0050  * If ttm_device_funcs->eviction_valuable returns true, then TTM will continue
0051  * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move
0052  * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.
0053  *
0054  * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to
0055  *  nofity when the BO is free to move. fence_add_callback --> enable_signaling
0056  *  --> amdgpu_amdkfd_fence.enable_signaling
0057  *
0058  * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce
0059  * user queues and signal fence. The work item will also start another delayed
0060  * work item to restore BOs
0061  */
0062
0063 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
0064                 struct mm_struct *mm,
0065                 struct svm_range_bo *svm_bo)
0066 {
0067     struct amdgpu_amdkfd_fence *fence;
0068
0069     fence = kzalloc(sizeof(*fence), GFP_KERNEL);
0070     if (fence == NULL)
0071         return NULL;
0072
0073     /* This reference gets released in amdkfd_fence_release */
0074     mmgrab(mm);
0075     fence->mm = mm;
0076     get_task_comm(fence->timeline_name, current);
0077     spin_lock_init(&fence->lock);
0078     fence->svm_bo = svm_bo;
0079     dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
0080            context, atomic_inc_return(&fence_seq));
0081
0082     return fence;
0083 }
0084
0085 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
0086 {
0087     struct amdgpu_amdkfd_fence *fence;
0088
0089     if (!f)
0090         return NULL;
0091
0092     fence = container_of(f, struct amdgpu_amdkfd_fence, base);
0093     if (fence && f->ops == &amdkfd_fence_ops)
0094         return fence;
0095
0096     return NULL;
0097 }
0098
0099 static const char *amdkfd_fence_get_driver_name(struct dma_fence *f)
0100 {
0101     return "amdgpu_amdkfd_fence";
0102 }
0103
0104 static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f)
0105 {
0106     struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
0107
0108     return fence->timeline_name;
0109 }
0110
0111 /**
0112  * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict
0113  *  a KFD BO and schedules a job to move the BO.
0114  *  If fence is already signaled return true.
0115  *  If fence is not signaled schedule a evict KFD process work item.
0116  *
0117  *  @f: dma_fence
0118  */
0119 static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
0120 {
0121     struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
0122
0123     if (!fence)
0124         return false;
0125
0126     if (dma_fence_is_signaled(f))
0127         return true;
0128
0129     if (!fence->svm_bo) {
0130         if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))
0131             return true;
0132     } else {
0133         if (!svm_range_schedule_evict_svm_bo(fence))
0134             return true;
0135     }
0136     return false;
0137 }
0138
0139 /**
0140  * amdkfd_fence_release - callback that fence can be freed
0141  *
0142  * @f: dma_fence
0143  *
0144  * This function is called when the reference count becomes zero.
0145  * Drops the mm_struct reference and RCU schedules freeing up the fence.
0146  */
0147 static void amdkfd_fence_release(struct dma_fence *f)
0148 {
0149     struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
0150
0151     /* Unconditionally signal the fence. The process is getting
0152      * terminated.
0153      */
0154     if (WARN_ON(!fence))
0155         return; /* Not an amdgpu_amdkfd_fence */
0156
0157     mmdrop(fence->mm);
0158     kfree_rcu(f, rcu);
0159 }
0160
0161 /**
0162  * amdkfd_fence_check_mm - Check whether to prevent eviction of @f by @mm
0163  *
0164  * @f: [IN] fence
0165  * @mm: [IN] mm that needs to be verified
0166  *
0167  * Check if @mm is same as that of the fence @f, if same return TRUE else
0168  * return FALSE.
0169  * For svm bo, which support vram overcommitment, always return FALSE.
0170  */
0171 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
0172 {
0173     struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);
0174
0175     if (!fence)
0176         return false;
0177     else if (fence->mm == mm  && !fence->svm_bo)
0178         return true;
0179
0180     return false;
0181 }
0182
0183 static const struct dma_fence_ops amdkfd_fence_ops = {
0184     .get_driver_name = amdkfd_fence_get_driver_name,
0185     .get_timeline_name = amdkfd_fence_get_timeline_name,
0186     .enable_signaling = amdkfd_fence_enable_signaling,
0187     .release = amdkfd_fence_release,
0188 };