amd/amdgpu/amdgpu_vm.h

0001 /*
0002  * Copyright 2016 Advanced Micro Devices, Inc.
0003  *
0004  * Permission is hereby granted, free of charge, to any person obtaining a
0005  * copy of this software and associated documentation files (the "Software"),
0006  * to deal in the Software without restriction, including without limitation
0007  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0008  * and/or sell copies of the Software, and to permit persons to whom the
0009  * Software is furnished to do so, subject to the following conditions:
0010  *
0011  * The above copyright notice and this permission notice shall be included in
0012  * all copies or substantial portions of the Software.
0013  *
0014  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0015  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0016  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0017  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0018  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0019  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0020  * OTHER DEALINGS IN THE SOFTWARE.
0021  *
0022  * Authors: Christian König
0023  */
0024 #ifndef __AMDGPU_VM_H__
0025 #define __AMDGPU_VM_H__
0026
0027 #include <linux/idr.h>
0028 #include <linux/kfifo.h>
0029 #include <linux/rbtree.h>
0030 #include <drm/gpu_scheduler.h>
0031 #include <drm/drm_file.h>
0032 #include <drm/ttm/ttm_bo_driver.h>
0033 #include <linux/sched/mm.h>
0034
0035 #include "amdgpu_sync.h"
0036 #include "amdgpu_ring.h"
0037 #include "amdgpu_ids.h"
0038
0039 struct amdgpu_bo_va;
0040 struct amdgpu_job;
0041 struct amdgpu_bo_list_entry;
0042 struct amdgpu_bo_vm;
0043
0044 /*
0045  * GPUVM handling
0046  */
0047
0048 /* Maximum number of PTEs the hardware can write with one command */
0049 #define AMDGPU_VM_MAX_UPDATE_SIZE   0x3FFFF
0050
0051 /* number of entries in page table */
0052 #define AMDGPU_VM_PTE_COUNT(adev) (1 << (adev)->vm_manager.block_size)
0053
0054 #define AMDGPU_PTE_VALID    (1ULL << 0)
0055 #define AMDGPU_PTE_SYSTEM   (1ULL << 1)
0056 #define AMDGPU_PTE_SNOOPED  (1ULL << 2)
0057
0058 /* RV+ */
0059 #define AMDGPU_PTE_TMZ      (1ULL << 3)
0060
0061 /* VI only */
0062 #define AMDGPU_PTE_EXECUTABLE   (1ULL << 4)
0063
0064 #define AMDGPU_PTE_READABLE (1ULL << 5)
0065 #define AMDGPU_PTE_WRITEABLE    (1ULL << 6)
0066
0067 #define AMDGPU_PTE_FRAG(x)  ((x & 0x1fULL) << 7)
0068
0069 /* TILED for VEGA10, reserved for older ASICs  */
0070 #define AMDGPU_PTE_PRT      (1ULL << 51)
0071
0072 /* PDE is handled as PTE for VEGA10 */
0073 #define AMDGPU_PDE_PTE      (1ULL << 54)
0074
0075 #define AMDGPU_PTE_LOG          (1ULL << 55)
0076
0077 /* PTE is handled as PDE for VEGA10 (Translate Further) */
0078 #define AMDGPU_PTE_TF       (1ULL << 56)
0079
0080 /* MALL noalloc for sienna_cichlid, reserved for older ASICs  */
0081 #define AMDGPU_PTE_NOALLOC  (1ULL << 58)
0082
0083 /* PDE Block Fragment Size for VEGA10 */
0084 #define AMDGPU_PDE_BFS(a)   ((uint64_t)a << 59)
0085
0086
0087 /* For GFX9 */
0088 #define AMDGPU_PTE_MTYPE_VG10(a)    ((uint64_t)(a) << 57)
0089 #define AMDGPU_PTE_MTYPE_VG10_MASK  AMDGPU_PTE_MTYPE_VG10(3ULL)
0090
0091 #define AMDGPU_MTYPE_NC 0
0092 #define AMDGPU_MTYPE_CC 2
0093
0094 #define AMDGPU_PTE_DEFAULT_ATC  (AMDGPU_PTE_SYSTEM      \
0095                                 | AMDGPU_PTE_SNOOPED    \
0096                                 | AMDGPU_PTE_EXECUTABLE \
0097                                 | AMDGPU_PTE_READABLE   \
0098                                 | AMDGPU_PTE_WRITEABLE  \
0099                                 | AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_CC))
0100
0101 /* gfx10 */
0102 #define AMDGPU_PTE_MTYPE_NV10(a)       ((uint64_t)(a) << 48)
0103 #define AMDGPU_PTE_MTYPE_NV10_MASK     AMDGPU_PTE_MTYPE_NV10(7ULL)
0104
0105 /* How to program VM fault handling */
0106 #define AMDGPU_VM_FAULT_STOP_NEVER  0
0107 #define AMDGPU_VM_FAULT_STOP_FIRST  1
0108 #define AMDGPU_VM_FAULT_STOP_ALWAYS 2
0109
0110 /* Reserve 4MB VRAM for page tables */
0111 #define AMDGPU_VM_RESERVED_VRAM     (8ULL << 20)
0112
0113 /* max number of VMHUB */
0114 #define AMDGPU_MAX_VMHUBS           3
0115 #define AMDGPU_GFXHUB_0             0
0116 #define AMDGPU_MMHUB_0              1
0117 #define AMDGPU_MMHUB_1              2
0118
0119 /* Reserve 2MB at top/bottom of address space for kernel use */
0120 #define AMDGPU_VA_RESERVED_SIZE         (2ULL << 20)
0121
0122 /* max vmids dedicated for process */
0123 #define AMDGPU_VM_MAX_RESERVED_VMID 1
0124
0125 /* See vm_update_mode */
0126 #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0)
0127 #define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1)
0128
0129 /* VMPT level enumerate, and the hiberachy is:
0130  * PDB2->PDB1->PDB0->PTB
0131  */
0132 enum amdgpu_vm_level {
0133     AMDGPU_VM_PDB2,
0134     AMDGPU_VM_PDB1,
0135     AMDGPU_VM_PDB0,
0136     AMDGPU_VM_PTB
0137 };
0138
0139 /* base structure for tracking BO usage in a VM */
0140 struct amdgpu_vm_bo_base {
0141     /* constant after initialization */
0142     struct amdgpu_vm        *vm;
0143     struct amdgpu_bo        *bo;
0144
0145     /* protected by bo being reserved */
0146     struct amdgpu_vm_bo_base    *next;
0147
0148     /* protected by spinlock */
0149     struct list_head        vm_status;
0150
0151     /* protected by the BO being reserved */
0152     bool                moved;
0153 };
0154
0155 /* provided by hw blocks that can write ptes, e.g., sdma */
0156 struct amdgpu_vm_pte_funcs {
0157     /* number of dw to reserve per operation */
0158     unsigned    copy_pte_num_dw;
0159
0160     /* copy pte entries from GART */
0161     void (*copy_pte)(struct amdgpu_ib *ib,
0162              uint64_t pe, uint64_t src,
0163              unsigned count);
0164
0165     /* write pte one entry at a time with addr mapping */
0166     void (*write_pte)(struct amdgpu_ib *ib, uint64_t pe,
0167               uint64_t value, unsigned count,
0168               uint32_t incr);
0169     /* for linear pte/pde updates without addr mapping */
0170     void (*set_pte_pde)(struct amdgpu_ib *ib,
0171                 uint64_t pe,
0172                 uint64_t addr, unsigned count,
0173                 uint32_t incr, uint64_t flags);
0174 };
0175
0176 struct amdgpu_task_info {
0177     char    process_name[TASK_COMM_LEN];
0178     char    task_name[TASK_COMM_LEN];
0179     pid_t   pid;
0180     pid_t   tgid;
0181 };
0182
0183 /**
0184  * struct amdgpu_vm_update_params
0185  *
0186  * Encapsulate some VM table update parameters to reduce
0187  * the number of function parameters
0188  *
0189  */
0190 struct amdgpu_vm_update_params {
0191
0192     /**
0193      * @adev: amdgpu device we do this update for
0194      */
0195     struct amdgpu_device *adev;
0196
0197     /**
0198      * @vm: optional amdgpu_vm we do this update for
0199      */
0200     struct amdgpu_vm *vm;
0201
0202     /**
0203      * @immediate: if changes should be made immediately
0204      */
0205     bool immediate;
0206
0207     /**
0208      * @unlocked: true if the root BO is not locked
0209      */
0210     bool unlocked;
0211
0212     /**
0213      * @pages_addr:
0214      *
0215      * DMA addresses to use for mapping
0216      */
0217     dma_addr_t *pages_addr;
0218
0219     /**
0220      * @job: job to used for hw submission
0221      */
0222     struct amdgpu_job *job;
0223
0224     /**
0225      * @num_dw_left: number of dw left for the IB
0226      */
0227     unsigned int num_dw_left;
0228
0229     /**
0230      * @table_freed: return true if page table is freed when updating
0231      */
0232     bool table_freed;
0233 };
0234
0235 struct amdgpu_vm_update_funcs {
0236     int (*map_table)(struct amdgpu_bo_vm *bo);
0237     int (*prepare)(struct amdgpu_vm_update_params *p, struct dma_resv *resv,
0238                enum amdgpu_sync_mode sync_mode);
0239     int (*update)(struct amdgpu_vm_update_params *p,
0240               struct amdgpu_bo_vm *bo, uint64_t pe, uint64_t addr,
0241               unsigned count, uint32_t incr, uint64_t flags);
0242     int (*commit)(struct amdgpu_vm_update_params *p,
0243               struct dma_fence **fence);
0244 };
0245
0246 struct amdgpu_vm {
0247     /* tree of virtual addresses mapped */
0248     struct rb_root_cached   va;
0249
0250     /* Lock to prevent eviction while we are updating page tables
0251      * use vm_eviction_lock/unlock(vm)
0252      */
0253     struct mutex        eviction_lock;
0254     bool            evicting;
0255     unsigned int        saved_flags;
0256
0257     /* BOs who needs a validation */
0258     struct list_head    evicted;
0259
0260     /* PT BOs which relocated and their parent need an update */
0261     struct list_head    relocated;
0262
0263     /* per VM BOs moved, but not yet updated in the PT */
0264     struct list_head    moved;
0265
0266     /* All BOs of this VM not currently in the state machine */
0267     struct list_head    idle;
0268
0269     /* regular invalidated BOs, but not yet updated in the PT */
0270     struct list_head    invalidated;
0271     spinlock_t      invalidated_lock;
0272
0273     /* BO mappings freed, but not yet updated in the PT */
0274     struct list_head    freed;
0275
0276     /* BOs which are invalidated, has been updated in the PTs */
0277     struct list_head        done;
0278
0279     /* contains the page directory */
0280     struct amdgpu_vm_bo_base     root;
0281     struct dma_fence    *last_update;
0282
0283     /* Scheduler entities for page table updates */
0284     struct drm_sched_entity immediate;
0285     struct drm_sched_entity delayed;
0286
0287     /* Last finished delayed update */
0288     atomic64_t      tlb_seq;
0289     struct dma_fence    *last_tlb_flush;
0290
0291     /* Last unlocked submission to the scheduler entities */
0292     struct dma_fence    *last_unlocked;
0293
0294     unsigned int        pasid;
0295     /* dedicated to vm */
0296     struct amdgpu_vmid  *reserved_vmid[AMDGPU_MAX_VMHUBS];
0297
0298     /* Flag to indicate if VM tables are updated by CPU or GPU (SDMA) */
0299     bool                    use_cpu_for_update;
0300
0301     /* Functions to use for VM table updates */
0302     const struct amdgpu_vm_update_funcs *update_funcs;
0303
0304     /* Flag to indicate ATS support from PTE for GFX9 */
0305     bool            pte_support_ats;
0306
0307     /* Up to 128 pending retry page faults */
0308     DECLARE_KFIFO(faults, u64, 128);
0309
0310     /* Points to the KFD process VM info */
0311     struct amdkfd_process_info *process_info;
0312
0313     /* List node in amdkfd_process_info.vm_list_head */
0314     struct list_head    vm_list_node;
0315
0316     /* Valid while the PD is reserved or fenced */
0317     uint64_t        pd_phys_addr;
0318
0319     /* Some basic info about the task */
0320     struct amdgpu_task_info task_info;
0321
0322     /* Store positions of group of BOs */
0323     struct ttm_lru_bulk_move lru_bulk_move;
0324     /* Flag to indicate if VM is used for compute */
0325     bool            is_compute_context;
0326 };
0327
0328 struct amdgpu_vm_manager {
0329     /* Handling of VMIDs */
0330     struct amdgpu_vmid_mgr          id_mgr[AMDGPU_MAX_VMHUBS];
0331     unsigned int                first_kfd_vmid;
0332     bool                    concurrent_flush;
0333
0334     /* Handling of VM fences */
0335     u64                 fence_context;
0336     unsigned                seqno[AMDGPU_MAX_RINGS];
0337
0338     uint64_t                max_pfn;
0339     uint32_t                num_level;
0340     uint32_t                block_size;
0341     uint32_t                fragment_size;
0342     enum amdgpu_vm_level            root_level;
0343     /* vram base address for page table entry  */
0344     u64                 vram_base_offset;
0345     /* vm pte handling */
0346     const struct amdgpu_vm_pte_funcs    *vm_pte_funcs;
0347     struct drm_gpu_scheduler        *vm_pte_scheds[AMDGPU_MAX_RINGS];
0348     unsigned                vm_pte_num_scheds;
0349     struct amdgpu_ring          *page_fault;
0350
0351     /* partial resident texture handling */
0352     spinlock_t              prt_lock;
0353     atomic_t                num_prt_users;
0354
0355     /* controls how VM page tables are updated for Graphics and Compute.
0356      * BIT0[= 0] Graphics updated by SDMA [= 1] by CPU
0357      * BIT1[= 0] Compute updated by SDMA [= 1] by CPU
0358      */
0359     int                 vm_update_mode;
0360
0361     /* PASID to VM mapping, will be used in interrupt context to
0362      * look up VM of a page fault
0363      */
0364     struct xarray               pasids;
0365 };
0366
0367 struct amdgpu_bo_va_mapping;
0368
0369 #define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))
0370 #define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
0371 #define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
0372
0373 extern const struct amdgpu_vm_update_funcs amdgpu_vm_cpu_funcs;
0374 extern const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs;
0375
0376 void amdgpu_vm_manager_init(struct amdgpu_device *adev);
0377 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
0378
0379 int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm,
0380             u32 pasid);
0381
0382 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout);
0383 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm);
0384 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
0385 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
0386 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
0387 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
0388              struct list_head *validated,
0389              struct amdgpu_bo_list_entry *entry);
0390 bool amdgpu_vm_ready(struct amdgpu_vm *vm);
0391 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
0392                   int (*callback)(void *p, struct amdgpu_bo *bo),
0393                   void *param);
0394 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync);
0395 int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
0396               struct amdgpu_vm *vm, bool immediate);
0397 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
0398               struct amdgpu_vm *vm,
0399               struct dma_fence **fence);
0400 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
0401                struct amdgpu_vm *vm);
0402 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
0403                 struct amdgpu_vm *vm, struct amdgpu_bo *bo);
0404 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
0405                bool immediate, bool unlocked, bool flush_tlb,
0406                struct dma_resv *resv, uint64_t start, uint64_t last,
0407                uint64_t flags, uint64_t offset, uint64_t vram_base,
0408                struct ttm_resource *res, dma_addr_t *pages_addr,
0409                struct dma_fence **fence);
0410 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
0411             struct amdgpu_bo_va *bo_va,
0412             bool clear);
0413 bool amdgpu_vm_evictable(struct amdgpu_bo *bo);
0414 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
0415                  struct amdgpu_bo *bo, bool evicted);
0416 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr);
0417 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
0418                        struct amdgpu_bo *bo);
0419 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
0420                       struct amdgpu_vm *vm,
0421                       struct amdgpu_bo *bo);
0422 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
0423              struct amdgpu_bo_va *bo_va,
0424              uint64_t addr, uint64_t offset,
0425              uint64_t size, uint64_t flags);
0426 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
0427                  struct amdgpu_bo_va *bo_va,
0428                  uint64_t addr, uint64_t offset,
0429                  uint64_t size, uint64_t flags);
0430 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
0431                struct amdgpu_bo_va *bo_va,
0432                uint64_t addr);
0433 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
0434                 struct amdgpu_vm *vm,
0435                 uint64_t saddr, uint64_t size);
0436 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
0437                              uint64_t addr);
0438 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket);
0439 void amdgpu_vm_bo_del(struct amdgpu_device *adev,
0440               struct amdgpu_bo_va *bo_va);
0441 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
0442                uint32_t fragment_size_default, unsigned max_level,
0443                unsigned max_bits);
0444 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
0445 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
0446                   struct amdgpu_job *job);
0447 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev);
0448
0449 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid,
0450                  struct amdgpu_task_info *task_info);
0451 bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
0452                 uint64_t addr, bool write_fault);
0453
0454 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm);
0455
0456 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
0457                 struct amdgpu_vm *vm);
0458 void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
0459                 uint64_t *gtt_mem, uint64_t *cpu_mem);
0460
0461 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
0462                struct amdgpu_bo_vm *vmbo, bool immediate);
0463 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm,
0464             int level, bool immediate, struct amdgpu_bo_vm **vmbo);
0465 void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm);
0466 bool amdgpu_vm_pt_is_root_clean(struct amdgpu_device *adev,
0467                 struct amdgpu_vm *vm);
0468
0469 int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
0470              struct amdgpu_vm_bo_base *entry);
0471 int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
0472               uint64_t start, uint64_t end,
0473               uint64_t dst, uint64_t flags);
0474
0475 #if defined(CONFIG_DEBUG_FS)
0476 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);
0477 #endif
0478
0479 /**
0480  * amdgpu_vm_tlb_seq - return tlb flush sequence number
0481  * @vm: the amdgpu_vm structure to query
0482  *
0483  * Returns the tlb flush sequence number which indicates that the VM TLBs needs
0484  * to be invalidated whenever the sequence number change.
0485  */
0486 static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm)
0487 {
0488     return atomic64_read(&vm->tlb_seq);
0489 }
0490
0491 #endif