amd/amdgpu/amdgpu_object.h

0001 /*
0002  * Copyright 2008 Advanced Micro Devices, Inc.
0003  * Copyright 2008 Red Hat Inc.
0004  * Copyright 2009 Jerome Glisse.
0005  *
0006  * Permission is hereby granted, free of charge, to any person obtaining a
0007  * copy of this software and associated documentation files (the "Software"),
0008  * to deal in the Software without restriction, including without limitation
0009  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0010  * and/or sell copies of the Software, and to permit persons to whom the
0011  * Software is furnished to do so, subject to the following conditions:
0012  *
0013  * The above copyright notice and this permission notice shall be included in
0014  * all copies or substantial portions of the Software.
0015  *
0016  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0017  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0018  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0019  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0020  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0021  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0022  * OTHER DEALINGS IN THE SOFTWARE.
0023  *
0024  * Authors: Dave Airlie
0025  *          Alex Deucher
0026  *          Jerome Glisse
0027  */
0028 #ifndef __AMDGPU_OBJECT_H__
0029 #define __AMDGPU_OBJECT_H__
0030
0031 #include <drm/amdgpu_drm.h>
0032 #include "amdgpu.h"
0033 #include "amdgpu_res_cursor.h"
0034
0035 #ifdef CONFIG_MMU_NOTIFIER
0036 #include <linux/mmu_notifier.h>
0037 #endif
0038
0039 #define AMDGPU_BO_INVALID_OFFSET    LONG_MAX
0040 #define AMDGPU_BO_MAX_PLACEMENTS    3
0041
0042 /* BO flag to indicate a KFD userptr BO */
0043 #define AMDGPU_AMDKFD_CREATE_USERPTR_BO (1ULL << 63)
0044
0045 #define to_amdgpu_bo_user(abo) container_of((abo), struct amdgpu_bo_user, bo)
0046 #define to_amdgpu_bo_vm(abo) container_of((abo), struct amdgpu_bo_vm, bo)
0047
0048 struct amdgpu_bo_param {
0049     unsigned long           size;
0050     int             byte_align;
0051     u32             bo_ptr_size;
0052     u32             domain;
0053     u32             preferred_domain;
0054     u64             flags;
0055     enum ttm_bo_type        type;
0056     bool                no_wait_gpu;
0057     struct dma_resv         *resv;
0058     void                (*destroy)(struct ttm_buffer_object *bo);
0059 };
0060
0061 /* bo virtual addresses in a vm */
0062 struct amdgpu_bo_va_mapping {
0063     struct amdgpu_bo_va     *bo_va;
0064     struct list_head        list;
0065     struct rb_node          rb;
0066     uint64_t            start;
0067     uint64_t            last;
0068     uint64_t            __subtree_last;
0069     uint64_t            offset;
0070     uint64_t            flags;
0071 };
0072
0073 /* User space allocated BO in a VM */
0074 struct amdgpu_bo_va {
0075     struct amdgpu_vm_bo_base    base;
0076
0077     /* protected by bo being reserved */
0078     unsigned            ref_count;
0079
0080     /* all other members protected by the VM PD being reserved */
0081     struct dma_fence            *last_pt_update;
0082
0083     /* mappings for this bo_va */
0084     struct list_head        invalids;
0085     struct list_head        valids;
0086
0087     /* If the mappings are cleared or filled */
0088     bool                cleared;
0089
0090     bool                is_xgmi;
0091 };
0092
0093 struct amdgpu_bo {
0094     /* Protected by tbo.reserved */
0095     u32             preferred_domains;
0096     u32             allowed_domains;
0097     struct ttm_place        placements[AMDGPU_BO_MAX_PLACEMENTS];
0098     struct ttm_placement        placement;
0099     struct ttm_buffer_object    tbo;
0100     struct ttm_bo_kmap_obj      kmap;
0101     u64             flags;
0102     /* per VM structure for page tables and with virtual addresses */
0103     struct amdgpu_vm_bo_base    *vm_bo;
0104     /* Constant after initialization */
0105     struct amdgpu_bo        *parent;
0106
0107 #ifdef CONFIG_MMU_NOTIFIER
0108     struct mmu_interval_notifier    notifier;
0109 #endif
0110     struct kgd_mem                  *kfd_bo;
0111 };
0112
0113 struct amdgpu_bo_user {
0114     struct amdgpu_bo        bo;
0115     u64             tiling_flags;
0116     u64             metadata_flags;
0117     void                *metadata;
0118     u32             metadata_size;
0119
0120 };
0121
0122 struct amdgpu_bo_vm {
0123     struct amdgpu_bo        bo;
0124     struct amdgpu_bo        *shadow;
0125     struct list_head        shadow_list;
0126     struct amdgpu_vm_bo_base        entries[];
0127 };
0128
0129 static inline struct amdgpu_bo *ttm_to_amdgpu_bo(struct ttm_buffer_object *tbo)
0130 {
0131     return container_of(tbo, struct amdgpu_bo, tbo);
0132 }
0133
0134 /**
0135  * amdgpu_mem_type_to_domain - return domain corresponding to mem_type
0136  * @mem_type:   ttm memory type
0137  *
0138  * Returns corresponding domain of the ttm mem_type
0139  */
0140 static inline unsigned amdgpu_mem_type_to_domain(u32 mem_type)
0141 {
0142     switch (mem_type) {
0143     case TTM_PL_VRAM:
0144         return AMDGPU_GEM_DOMAIN_VRAM;
0145     case TTM_PL_TT:
0146         return AMDGPU_GEM_DOMAIN_GTT;
0147     case TTM_PL_SYSTEM:
0148         return AMDGPU_GEM_DOMAIN_CPU;
0149     case AMDGPU_PL_GDS:
0150         return AMDGPU_GEM_DOMAIN_GDS;
0151     case AMDGPU_PL_GWS:
0152         return AMDGPU_GEM_DOMAIN_GWS;
0153     case AMDGPU_PL_OA:
0154         return AMDGPU_GEM_DOMAIN_OA;
0155     default:
0156         break;
0157     }
0158     return 0;
0159 }
0160
0161 /**
0162  * amdgpu_bo_reserve - reserve bo
0163  * @bo:     bo structure
0164  * @no_intr:    don't return -ERESTARTSYS on pending signal
0165  *
0166  * Returns:
0167  * -ERESTARTSYS: A wait for the buffer to become unreserved was interrupted by
0168  * a signal. Release all buffer reservations and return to user-space.
0169  */
0170 static inline int amdgpu_bo_reserve(struct amdgpu_bo *bo, bool no_intr)
0171 {
0172     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
0173     int r;
0174
0175     r = ttm_bo_reserve(&bo->tbo, !no_intr, false, NULL);
0176     if (unlikely(r != 0)) {
0177         if (r != -ERESTARTSYS)
0178             dev_err(adev->dev, "%p reserve failed\n", bo);
0179         return r;
0180     }
0181     return 0;
0182 }
0183
0184 static inline void amdgpu_bo_unreserve(struct amdgpu_bo *bo)
0185 {
0186     ttm_bo_unreserve(&bo->tbo);
0187 }
0188
0189 static inline unsigned long amdgpu_bo_size(struct amdgpu_bo *bo)
0190 {
0191     return bo->tbo.base.size;
0192 }
0193
0194 static inline unsigned amdgpu_bo_ngpu_pages(struct amdgpu_bo *bo)
0195 {
0196     return bo->tbo.base.size / AMDGPU_GPU_PAGE_SIZE;
0197 }
0198
0199 static inline unsigned amdgpu_bo_gpu_page_alignment(struct amdgpu_bo *bo)
0200 {
0201     return (bo->tbo.page_alignment << PAGE_SHIFT) / AMDGPU_GPU_PAGE_SIZE;
0202 }
0203
0204 /**
0205  * amdgpu_bo_mmap_offset - return mmap offset of bo
0206  * @bo: amdgpu object for which we query the offset
0207  *
0208  * Returns mmap offset of the object.
0209  */
0210 static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo)
0211 {
0212     return drm_vma_node_offset_addr(&bo->tbo.base.vma_node);
0213 }
0214
0215 /**
0216  * amdgpu_bo_in_cpu_visible_vram - check if BO is (partly) in visible VRAM
0217  */
0218 static inline bool amdgpu_bo_in_cpu_visible_vram(struct amdgpu_bo *bo)
0219 {
0220     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
0221     struct amdgpu_res_cursor cursor;
0222
0223     if (bo->tbo.resource->mem_type != TTM_PL_VRAM)
0224         return false;
0225
0226     amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor);
0227     while (cursor.remaining) {
0228         if (cursor.start < adev->gmc.visible_vram_size)
0229             return true;
0230
0231         amdgpu_res_next(&cursor, cursor.size);
0232     }
0233
0234     return false;
0235 }
0236
0237 /**
0238  * amdgpu_bo_explicit_sync - return whether the bo is explicitly synced
0239  */
0240 static inline bool amdgpu_bo_explicit_sync(struct amdgpu_bo *bo)
0241 {
0242     return bo->flags & AMDGPU_GEM_CREATE_EXPLICIT_SYNC;
0243 }
0244
0245 /**
0246  * amdgpu_bo_encrypted - test if the BO is encrypted
0247  * @bo: pointer to a buffer object
0248  *
0249  * Return true if the buffer object is encrypted, false otherwise.
0250  */
0251 static inline bool amdgpu_bo_encrypted(struct amdgpu_bo *bo)
0252 {
0253     return bo->flags & AMDGPU_GEM_CREATE_ENCRYPTED;
0254 }
0255
0256 /**
0257  * amdgpu_bo_shadowed - check if the BO is shadowed
0258  *
0259  * @bo: BO to be tested.
0260  *
0261  * Returns:
0262  * NULL if not shadowed or else return a BO pointer.
0263  */
0264 static inline struct amdgpu_bo *amdgpu_bo_shadowed(struct amdgpu_bo *bo)
0265 {
0266     if (bo->tbo.type == ttm_bo_type_kernel)
0267         return to_amdgpu_bo_vm(bo)->shadow;
0268
0269     return NULL;
0270 }
0271
0272 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
0273 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
0274
0275 int amdgpu_bo_create(struct amdgpu_device *adev,
0276              struct amdgpu_bo_param *bp,
0277              struct amdgpu_bo **bo_ptr);
0278 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
0279                   unsigned long size, int align,
0280                   u32 domain, struct amdgpu_bo **bo_ptr,
0281                   u64 *gpu_addr, void **cpu_addr);
0282 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
0283                 unsigned long size, int align,
0284                 u32 domain, struct amdgpu_bo **bo_ptr,
0285                 u64 *gpu_addr, void **cpu_addr);
0286 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
0287                    uint64_t offset, uint64_t size, uint32_t domain,
0288                    struct amdgpu_bo **bo_ptr, void **cpu_addr);
0289 int amdgpu_bo_create_user(struct amdgpu_device *adev,
0290               struct amdgpu_bo_param *bp,
0291               struct amdgpu_bo_user **ubo_ptr);
0292 int amdgpu_bo_create_vm(struct amdgpu_device *adev,
0293             struct amdgpu_bo_param *bp,
0294             struct amdgpu_bo_vm **ubo_ptr);
0295 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
0296                void **cpu_addr);
0297 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr);
0298 void *amdgpu_bo_kptr(struct amdgpu_bo *bo);
0299 void amdgpu_bo_kunmap(struct amdgpu_bo *bo);
0300 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo);
0301 void amdgpu_bo_unref(struct amdgpu_bo **bo);
0302 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain);
0303 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
0304                  u64 min_offset, u64 max_offset);
0305 void amdgpu_bo_unpin(struct amdgpu_bo *bo);
0306 int amdgpu_bo_init(struct amdgpu_device *adev);
0307 void amdgpu_bo_fini(struct amdgpu_device *adev);
0308 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags);
0309 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags);
0310 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
0311                 uint32_t metadata_size, uint64_t flags);
0312 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
0313                size_t buffer_size, uint32_t *metadata_size,
0314                uint64_t *flags);
0315 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
0316                bool evict,
0317                struct ttm_resource *new_mem);
0318 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo);
0319 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
0320 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
0321              bool shared);
0322 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
0323                  enum amdgpu_sync_mode sync_mode, void *owner,
0324                  bool intr);
0325 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr);
0326 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo);
0327 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo);
0328 void amdgpu_bo_get_memory(struct amdgpu_bo *bo, uint64_t *vram_mem,
0329                 uint64_t *gtt_mem, uint64_t *cpu_mem);
0330 void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm *vmbo);
0331 int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow,
0332                  struct dma_fence **fence);
0333 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
0334                         uint32_t domain);
0335
0336 /*
0337  * sub allocation
0338  */
0339
0340 static inline uint64_t amdgpu_sa_bo_gpu_addr(struct amdgpu_sa_bo *sa_bo)
0341 {
0342     return sa_bo->manager->gpu_addr + sa_bo->soffset;
0343 }
0344
0345 static inline void * amdgpu_sa_bo_cpu_addr(struct amdgpu_sa_bo *sa_bo)
0346 {
0347     return sa_bo->manager->cpu_ptr + sa_bo->soffset;
0348 }
0349
0350 int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev,
0351                      struct amdgpu_sa_manager *sa_manager,
0352                      unsigned size, u32 align, u32 domain);
0353 void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev,
0354                       struct amdgpu_sa_manager *sa_manager);
0355 int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev,
0356                       struct amdgpu_sa_manager *sa_manager);
0357 int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager,
0358              struct amdgpu_sa_bo **sa_bo,
0359              unsigned size, unsigned align);
0360 void amdgpu_sa_bo_free(struct amdgpu_device *adev,
0361                   struct amdgpu_sa_bo **sa_bo,
0362                   struct dma_fence *fence);
0363 #if defined(CONFIG_DEBUG_FS)
0364 void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
0365                      struct seq_file *m);
0366 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m);
0367 #endif
0368 void amdgpu_debugfs_sa_init(struct amdgpu_device *adev);
0369
0370 bool amdgpu_bo_support_uswc(u64 bo_flags);
0371
0372
0373 #endif