amd/amdgpu/amdgpu_amdkfd.h

0001 /*
0002  * Copyright 2014 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  */
0022
0023 /* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */
0024
0025 #ifndef AMDGPU_AMDKFD_H_INCLUDED
0026 #define AMDGPU_AMDKFD_H_INCLUDED
0027
0028 #include <linux/types.h>
0029 #include <linux/mm.h>
0030 #include <linux/kthread.h>
0031 #include <linux/workqueue.h>
0032 #include <kgd_kfd_interface.h>
0033 #include <drm/ttm/ttm_execbuf_util.h>
0034 #include "amdgpu_sync.h"
0035 #include "amdgpu_vm.h"
0036
0037 extern uint64_t amdgpu_amdkfd_total_mem_size;
0038
0039 enum TLB_FLUSH_TYPE {
0040     TLB_FLUSH_LEGACY = 0,
0041     TLB_FLUSH_LIGHTWEIGHT,
0042     TLB_FLUSH_HEAVYWEIGHT
0043 };
0044
0045 struct amdgpu_device;
0046
0047 enum kfd_mem_attachment_type {
0048     KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
0049     KFD_MEM_ATT_USERPTR,    /* SG bo to DMA map pages from a userptr bo */
0050     KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
0051     KFD_MEM_ATT_SG      /* Tag to DMA map SG BOs */
0052 };
0053
0054 struct kfd_mem_attachment {
0055     struct list_head list;
0056     enum kfd_mem_attachment_type type;
0057     bool is_mapped;
0058     struct amdgpu_bo_va *bo_va;
0059     struct amdgpu_device *adev;
0060     uint64_t va;
0061     uint64_t pte_flags;
0062 };
0063
0064 struct kgd_mem {
0065     struct mutex lock;
0066     struct amdgpu_bo *bo;
0067     struct dma_buf *dmabuf;
0068     struct list_head attachments;
0069     /* protected by amdkfd_process_info.lock */
0070     struct ttm_validate_buffer validate_list;
0071     struct ttm_validate_buffer resv_list;
0072     uint32_t domain;
0073     unsigned int mapped_to_gpu_memory;
0074     uint64_t va;
0075
0076     uint32_t alloc_flags;
0077
0078     atomic_t invalid;
0079     struct amdkfd_process_info *process_info;
0080
0081     struct amdgpu_sync sync;
0082
0083     bool aql_queue;
0084     bool is_imported;
0085 };
0086
0087 /* KFD Memory Eviction */
0088 struct amdgpu_amdkfd_fence {
0089     struct dma_fence base;
0090     struct mm_struct *mm;
0091     spinlock_t lock;
0092     char timeline_name[TASK_COMM_LEN];
0093     struct svm_range_bo *svm_bo;
0094 };
0095
0096 struct amdgpu_kfd_dev {
0097     struct kfd_dev *dev;
0098     uint64_t vram_used;
0099     uint64_t vram_used_aligned;
0100     bool init_complete;
0101     struct work_struct reset_work;
0102 };
0103
0104 enum kgd_engine_type {
0105     KGD_ENGINE_PFP = 1,
0106     KGD_ENGINE_ME,
0107     KGD_ENGINE_CE,
0108     KGD_ENGINE_MEC1,
0109     KGD_ENGINE_MEC2,
0110     KGD_ENGINE_RLC,
0111     KGD_ENGINE_SDMA1,
0112     KGD_ENGINE_SDMA2,
0113     KGD_ENGINE_MAX
0114 };
0115
0116
0117 struct amdkfd_process_info {
0118     /* List head of all VMs that belong to a KFD process */
0119     struct list_head vm_list_head;
0120     /* List head for all KFD BOs that belong to a KFD process. */
0121     struct list_head kfd_bo_list;
0122     /* List of userptr BOs that are valid or invalid */
0123     struct list_head userptr_valid_list;
0124     struct list_head userptr_inval_list;
0125     /* Lock to protect kfd_bo_list */
0126     struct mutex lock;
0127
0128     /* Number of VMs */
0129     unsigned int n_vms;
0130     /* Eviction Fence */
0131     struct amdgpu_amdkfd_fence *eviction_fence;
0132
0133     /* MMU-notifier related fields */
0134     atomic_t evicted_bos;
0135     struct delayed_work restore_userptr_work;
0136     struct pid *pid;
0137     bool block_mmu_notifications;
0138 };
0139
0140 int amdgpu_amdkfd_init(void);
0141 void amdgpu_amdkfd_fini(void);
0142
0143 void amdgpu_amdkfd_suspend(struct amdgpu_device *adev, bool run_pm);
0144 int amdgpu_amdkfd_resume_iommu(struct amdgpu_device *adev);
0145 int amdgpu_amdkfd_resume(struct amdgpu_device *adev, bool run_pm);
0146 void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
0147             const void *ih_ring_entry);
0148 void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
0149 void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
0150 void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev);
0151 int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
0152                 enum kgd_engine_type engine,
0153                 uint32_t vmid, uint64_t gpu_addr,
0154                 uint32_t *ib_cmd, uint32_t ib_len);
0155 void amdgpu_amdkfd_set_compute_idle(struct amdgpu_device *adev, bool idle);
0156 bool amdgpu_amdkfd_have_atomics_support(struct amdgpu_device *adev);
0157 int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct amdgpu_device *adev,
0158                 uint16_t vmid);
0159 int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
0160                 uint16_t pasid, enum TLB_FLUSH_TYPE flush_type);
0161
0162 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
0163
0164 int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);
0165
0166 int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);
0167
0168 void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev);
0169
0170 int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
0171                     int queue_bit);
0172
0173 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
0174                 struct mm_struct *mm,
0175                 struct svm_range_bo *svm_bo);
0176 #if defined(CONFIG_DEBUG_FS)
0177 int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data);
0178 #endif
0179 #if IS_ENABLED(CONFIG_HSA_AMD)
0180 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
0181 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);
0182 int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo);
0183 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
0184 #else
0185 static inline
0186 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
0187 {
0188     return false;
0189 }
0190
0191 static inline
0192 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
0193 {
0194     return NULL;
0195 }
0196
0197 static inline
0198 int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)
0199 {
0200     return 0;
0201 }
0202
0203 static inline
0204 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm)
0205 {
0206     return 0;
0207 }
0208 #endif
0209 /* Shared API */
0210 int amdgpu_amdkfd_alloc_gtt_mem(struct amdgpu_device *adev, size_t size,
0211                 void **mem_obj, uint64_t *gpu_addr,
0212                 void **cpu_ptr, bool mqd_gfx9);
0213 void amdgpu_amdkfd_free_gtt_mem(struct amdgpu_device *adev, void *mem_obj);
0214 int amdgpu_amdkfd_alloc_gws(struct amdgpu_device *adev, size_t size,
0215                 void **mem_obj);
0216 void amdgpu_amdkfd_free_gws(struct amdgpu_device *adev, void *mem_obj);
0217 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem);
0218 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem);
0219 uint32_t amdgpu_amdkfd_get_fw_version(struct amdgpu_device *adev,
0220                       enum kgd_engine_type type);
0221 void amdgpu_amdkfd_get_local_mem_info(struct amdgpu_device *adev,
0222                       struct kfd_local_mem_info *mem_info);
0223 uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct amdgpu_device *adev);
0224
0225 uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev);
0226 void amdgpu_amdkfd_get_cu_info(struct amdgpu_device *adev,
0227                    struct kfd_cu_info *cu_info);
0228 int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd,
0229                   struct amdgpu_device **dmabuf_adev,
0230                   uint64_t *bo_size, void *metadata_buffer,
0231                   size_t buffer_size, uint32_t *metadata_size,
0232                   uint32_t *flags);
0233 uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct amdgpu_device *dst,
0234                       struct amdgpu_device *src);
0235 int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct amdgpu_device *dst,
0236                         struct amdgpu_device *src,
0237                         bool is_min);
0238 int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct amdgpu_device *adev, bool is_min);
0239
0240 /* Read user wptr from a specified user address space with page fault
0241  * disabled. The memory must be pinned and mapped to the hardware when
0242  * this is called in hqd_load functions, so it should never fault in
0243  * the first place. This resolves a circular lock dependency involving
0244  * four locks, including the DQM lock and mmap_lock.
0245  */
0246 #define read_user_wptr(mmptr, wptr, dst)                \
0247     ({                              \
0248         bool valid = false;                 \
0249         if ((mmptr) && (wptr)) {                \
0250             pagefault_disable();                \
0251             if ((mmptr) == current->mm) {           \
0252                 valid = !get_user((dst), (wptr));   \
0253             } else if (current->flags & PF_KTHREAD) {   \
0254                 kthread_use_mm(mmptr);          \
0255                 valid = !get_user((dst), (wptr));   \
0256                 kthread_unuse_mm(mmptr);        \
0257             }                       \
0258             pagefault_enable();             \
0259         }                           \
0260         valid;                          \
0261     })
0262
0263 /* GPUVM API */
0264 #define drm_priv_to_vm(drm_priv)                    \
0265     (&((struct amdgpu_fpriv *)                  \
0266         ((struct drm_file *)(drm_priv))->driver_priv)->vm)
0267
0268 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
0269                     struct file *filp, u32 pasid,
0270                     void **process_info,
0271                     struct dma_fence **ef);
0272 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
0273                     void *drm_priv);
0274 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
0275 size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev);
0276 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
0277         struct amdgpu_device *adev, uint64_t va, uint64_t size,
0278         void *drm_priv, struct kgd_mem **mem,
0279         uint64_t *offset, uint32_t flags, bool criu_resume);
0280 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
0281         struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv,
0282         uint64_t *size);
0283 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(struct amdgpu_device *adev,
0284                       struct kgd_mem *mem, void *drm_priv);
0285 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
0286         struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv);
0287 int amdgpu_amdkfd_gpuvm_sync_memory(
0288         struct amdgpu_device *adev, struct kgd_mem *mem, bool intr);
0289 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem,
0290                          void **kptr, uint64_t *size);
0291 void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem);
0292
0293 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_device *adev, struct amdgpu_bo *bo);
0294
0295 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
0296                         struct dma_fence **ef);
0297 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
0298                           struct kfd_vm_fault_info *info);
0299 int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
0300                       struct dma_buf *dmabuf,
0301                       uint64_t va, void *drm_priv,
0302                       struct kgd_mem **mem, uint64_t *size,
0303                       uint64_t *mmap_offset);
0304 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev,
0305                 struct tile_config *config);
0306 void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev,
0307                 bool reset);
0308 bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem);
0309 void amdgpu_amdkfd_block_mmu_notifications(void *p);
0310 int amdgpu_amdkfd_criu_resume(void *p);
0311 bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev);
0312 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
0313         uint64_t size, u32 alloc_flag);
0314 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
0315         uint64_t size, u32 alloc_flag);
0316
0317 #if IS_ENABLED(CONFIG_HSA_AMD)
0318 void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
0319 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
0320                 struct amdgpu_vm *vm);
0321
0322 /**
0323  * @amdgpu_amdkfd_release_notify() - Notify KFD when GEM object is released
0324  *
0325  * Allows KFD to release its resources associated with the GEM object.
0326  */
0327 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo);
0328 void amdgpu_amdkfd_reserve_system_mem(uint64_t size);
0329 #else
0330 static inline
0331 void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
0332 {
0333 }
0334
0335 static inline
0336 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
0337                     struct amdgpu_vm *vm)
0338 {
0339 }
0340
0341 static inline
0342 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
0343 {
0344 }
0345 #endif
0346 /* KGD2KFD callbacks */
0347 int kgd2kfd_quiesce_mm(struct mm_struct *mm, uint32_t trigger);
0348 int kgd2kfd_resume_mm(struct mm_struct *mm);
0349 int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
0350                         struct dma_fence *fence);
0351 #if IS_ENABLED(CONFIG_HSA_AMD)
0352 int kgd2kfd_init(void);
0353 void kgd2kfd_exit(void);
0354 struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf);
0355 bool kgd2kfd_device_init(struct kfd_dev *kfd,
0356              struct drm_device *ddev,
0357              const struct kgd2kfd_shared_resources *gpu_resources);
0358 void kgd2kfd_device_exit(struct kfd_dev *kfd);
0359 void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm);
0360 int kgd2kfd_resume_iommu(struct kfd_dev *kfd);
0361 int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm);
0362 int kgd2kfd_pre_reset(struct kfd_dev *kfd);
0363 int kgd2kfd_post_reset(struct kfd_dev *kfd);
0364 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
0365 void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd);
0366 void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask);
0367 #else
0368 static inline int kgd2kfd_init(void)
0369 {
0370     return -ENOENT;
0371 }
0372
0373 static inline void kgd2kfd_exit(void)
0374 {
0375 }
0376
0377 static inline
0378 struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
0379 {
0380     return NULL;
0381 }
0382
0383 static inline
0384 bool kgd2kfd_device_init(struct kfd_dev *kfd, struct drm_device *ddev,
0385                 const struct kgd2kfd_shared_resources *gpu_resources)
0386 {
0387     return false;
0388 }
0389
0390 static inline void kgd2kfd_device_exit(struct kfd_dev *kfd)
0391 {
0392 }
0393
0394 static inline void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)
0395 {
0396 }
0397
0398 static int __maybe_unused kgd2kfd_resume_iommu(struct kfd_dev *kfd)
0399 {
0400     return 0;
0401 }
0402
0403 static inline int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)
0404 {
0405     return 0;
0406 }
0407
0408 static inline int kgd2kfd_pre_reset(struct kfd_dev *kfd)
0409 {
0410     return 0;
0411 }
0412
0413 static inline int kgd2kfd_post_reset(struct kfd_dev *kfd)
0414 {
0415     return 0;
0416 }
0417
0418 static inline
0419 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
0420 {
0421 }
0422
0423 static inline
0424 void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
0425 {
0426 }
0427
0428 static inline
0429 void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask)
0430 {
0431 }
0432 #endif
0433 #endif /* AMDGPU_AMDKFD_H_INCLUDED */