Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Copyright 2009 Jerome Glisse.
0003  * All Rights Reserved.
0004  *
0005  * Permission is hereby granted, free of charge, to any person obtaining a
0006  * copy of this software and associated documentation files (the
0007  * "Software"), to deal in the Software without restriction, including
0008  * without limitation the rights to use, copy, modify, merge, publish,
0009  * distribute, sub license, and/or sell copies of the Software, and to
0010  * permit persons to whom the Software is furnished to do so, subject to
0011  * the following conditions:
0012  *
0013  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0014  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0015  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
0016  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
0017  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
0018  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
0019  * USE OR OTHER DEALINGS IN THE SOFTWARE.
0020  *
0021  * The above copyright notice and this permission notice (including the
0022  * next paragraph) shall be included in all copies or substantial portions
0023  * of the Software.
0024  *
0025  */
0026 /*
0027  * Authors:
0028  *    Jerome Glisse <glisse@freedesktop.org>
0029  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
0030  *    Dave Airlie
0031  */
0032 #include <linux/list.h>
0033 #include <linux/slab.h>
0034 #include <linux/dma-buf.h>
0035 
0036 #include <drm/drm_drv.h>
0037 #include <drm/amdgpu_drm.h>
0038 #include <drm/drm_cache.h>
0039 #include "amdgpu.h"
0040 #include "amdgpu_trace.h"
0041 #include "amdgpu_amdkfd.h"
0042 
0043 /**
0044  * DOC: amdgpu_object
0045  *
0046  * This defines the interfaces to operate on an &amdgpu_bo buffer object which
0047  * represents memory used by driver (VRAM, system memory, etc.). The driver
0048  * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
0049  * to create/destroy/set buffer object which are then managed by the kernel TTM
0050  * memory manager.
0051  * The interfaces are also used internally by kernel clients, including gfx,
0052  * uvd, etc. for kernel managed allocations used by the GPU.
0053  *
0054  */
0055 
0056 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
0057 {
0058     struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
0059 
0060     amdgpu_bo_kunmap(bo);
0061 
0062     if (bo->tbo.base.import_attach)
0063         drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);
0064     drm_gem_object_release(&bo->tbo.base);
0065     amdgpu_bo_unref(&bo->parent);
0066     kvfree(bo);
0067 }
0068 
0069 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
0070 {
0071     struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
0072     struct amdgpu_bo_user *ubo;
0073 
0074     ubo = to_amdgpu_bo_user(bo);
0075     kfree(ubo->metadata);
0076     amdgpu_bo_destroy(tbo);
0077 }
0078 
0079 static void amdgpu_bo_vm_destroy(struct ttm_buffer_object *tbo)
0080 {
0081     struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
0082     struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
0083     struct amdgpu_bo_vm *vmbo;
0084 
0085     vmbo = to_amdgpu_bo_vm(bo);
0086     /* in case amdgpu_device_recover_vram got NULL of bo->parent */
0087     if (!list_empty(&vmbo->shadow_list)) {
0088         mutex_lock(&adev->shadow_list_lock);
0089         list_del_init(&vmbo->shadow_list);
0090         mutex_unlock(&adev->shadow_list_lock);
0091     }
0092 
0093     amdgpu_bo_destroy(tbo);
0094 }
0095 
0096 /**
0097  * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
0098  * @bo: buffer object to be checked
0099  *
0100  * Uses destroy function associated with the object to determine if this is
0101  * an &amdgpu_bo.
0102  *
0103  * Returns:
0104  * true if the object belongs to &amdgpu_bo, false if not.
0105  */
0106 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
0107 {
0108     if (bo->destroy == &amdgpu_bo_destroy ||
0109         bo->destroy == &amdgpu_bo_user_destroy ||
0110         bo->destroy == &amdgpu_bo_vm_destroy)
0111         return true;
0112 
0113     return false;
0114 }
0115 
0116 /**
0117  * amdgpu_bo_placement_from_domain - set buffer's placement
0118  * @abo: &amdgpu_bo buffer object whose placement is to be set
0119  * @domain: requested domain
0120  *
0121  * Sets buffer's placement according to requested domain and the buffer's
0122  * flags.
0123  */
0124 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
0125 {
0126     struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
0127     struct ttm_placement *placement = &abo->placement;
0128     struct ttm_place *places = abo->placements;
0129     u64 flags = abo->flags;
0130     u32 c = 0;
0131 
0132     if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
0133         unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
0134 
0135         places[c].fpfn = 0;
0136         places[c].lpfn = 0;
0137         places[c].mem_type = TTM_PL_VRAM;
0138         places[c].flags = 0;
0139 
0140         if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
0141             places[c].lpfn = visible_pfn;
0142         else
0143             places[c].flags |= TTM_PL_FLAG_TOPDOWN;
0144 
0145         if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
0146             places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
0147         c++;
0148     }
0149 
0150     if (domain & AMDGPU_GEM_DOMAIN_GTT) {
0151         places[c].fpfn = 0;
0152         places[c].lpfn = 0;
0153         places[c].mem_type =
0154             abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
0155             AMDGPU_PL_PREEMPT : TTM_PL_TT;
0156         places[c].flags = 0;
0157         c++;
0158     }
0159 
0160     if (domain & AMDGPU_GEM_DOMAIN_CPU) {
0161         places[c].fpfn = 0;
0162         places[c].lpfn = 0;
0163         places[c].mem_type = TTM_PL_SYSTEM;
0164         places[c].flags = 0;
0165         c++;
0166     }
0167 
0168     if (domain & AMDGPU_GEM_DOMAIN_GDS) {
0169         places[c].fpfn = 0;
0170         places[c].lpfn = 0;
0171         places[c].mem_type = AMDGPU_PL_GDS;
0172         places[c].flags = 0;
0173         c++;
0174     }
0175 
0176     if (domain & AMDGPU_GEM_DOMAIN_GWS) {
0177         places[c].fpfn = 0;
0178         places[c].lpfn = 0;
0179         places[c].mem_type = AMDGPU_PL_GWS;
0180         places[c].flags = 0;
0181         c++;
0182     }
0183 
0184     if (domain & AMDGPU_GEM_DOMAIN_OA) {
0185         places[c].fpfn = 0;
0186         places[c].lpfn = 0;
0187         places[c].mem_type = AMDGPU_PL_OA;
0188         places[c].flags = 0;
0189         c++;
0190     }
0191 
0192     if (!c) {
0193         places[c].fpfn = 0;
0194         places[c].lpfn = 0;
0195         places[c].mem_type = TTM_PL_SYSTEM;
0196         places[c].flags = 0;
0197         c++;
0198     }
0199 
0200     BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS);
0201 
0202     placement->num_placement = c;
0203     placement->placement = places;
0204 
0205     placement->num_busy_placement = c;
0206     placement->busy_placement = places;
0207 }
0208 
0209 /**
0210  * amdgpu_bo_create_reserved - create reserved BO for kernel use
0211  *
0212  * @adev: amdgpu device object
0213  * @size: size for the new BO
0214  * @align: alignment for the new BO
0215  * @domain: where to place it
0216  * @bo_ptr: used to initialize BOs in structures
0217  * @gpu_addr: GPU addr of the pinned BO
0218  * @cpu_addr: optional CPU address mapping
0219  *
0220  * Allocates and pins a BO for kernel internal use, and returns it still
0221  * reserved.
0222  *
0223  * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
0224  *
0225  * Returns:
0226  * 0 on success, negative error code otherwise.
0227  */
0228 int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
0229                   unsigned long size, int align,
0230                   u32 domain, struct amdgpu_bo **bo_ptr,
0231                   u64 *gpu_addr, void **cpu_addr)
0232 {
0233     struct amdgpu_bo_param bp;
0234     bool free = false;
0235     int r;
0236 
0237     if (!size) {
0238         amdgpu_bo_unref(bo_ptr);
0239         return 0;
0240     }
0241 
0242     memset(&bp, 0, sizeof(bp));
0243     bp.size = size;
0244     bp.byte_align = align;
0245     bp.domain = domain;
0246     bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
0247         : AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
0248     bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
0249     bp.type = ttm_bo_type_kernel;
0250     bp.resv = NULL;
0251     bp.bo_ptr_size = sizeof(struct amdgpu_bo);
0252 
0253     if (!*bo_ptr) {
0254         r = amdgpu_bo_create(adev, &bp, bo_ptr);
0255         if (r) {
0256             dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
0257                 r);
0258             return r;
0259         }
0260         free = true;
0261     }
0262 
0263     r = amdgpu_bo_reserve(*bo_ptr, false);
0264     if (r) {
0265         dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
0266         goto error_free;
0267     }
0268 
0269     r = amdgpu_bo_pin(*bo_ptr, domain);
0270     if (r) {
0271         dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
0272         goto error_unreserve;
0273     }
0274 
0275     r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
0276     if (r) {
0277         dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
0278         goto error_unpin;
0279     }
0280 
0281     if (gpu_addr)
0282         *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
0283 
0284     if (cpu_addr) {
0285         r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
0286         if (r) {
0287             dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
0288             goto error_unpin;
0289         }
0290     }
0291 
0292     return 0;
0293 
0294 error_unpin:
0295     amdgpu_bo_unpin(*bo_ptr);
0296 error_unreserve:
0297     amdgpu_bo_unreserve(*bo_ptr);
0298 
0299 error_free:
0300     if (free)
0301         amdgpu_bo_unref(bo_ptr);
0302 
0303     return r;
0304 }
0305 
0306 /**
0307  * amdgpu_bo_create_kernel - create BO for kernel use
0308  *
0309  * @adev: amdgpu device object
0310  * @size: size for the new BO
0311  * @align: alignment for the new BO
0312  * @domain: where to place it
0313  * @bo_ptr:  used to initialize BOs in structures
0314  * @gpu_addr: GPU addr of the pinned BO
0315  * @cpu_addr: optional CPU address mapping
0316  *
0317  * Allocates and pins a BO for kernel internal use.
0318  *
0319  * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
0320  *
0321  * Returns:
0322  * 0 on success, negative error code otherwise.
0323  */
0324 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
0325                 unsigned long size, int align,
0326                 u32 domain, struct amdgpu_bo **bo_ptr,
0327                 u64 *gpu_addr, void **cpu_addr)
0328 {
0329     int r;
0330 
0331     r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
0332                       gpu_addr, cpu_addr);
0333 
0334     if (r)
0335         return r;
0336 
0337     if (*bo_ptr)
0338         amdgpu_bo_unreserve(*bo_ptr);
0339 
0340     return 0;
0341 }
0342 
0343 /**
0344  * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
0345  *
0346  * @adev: amdgpu device object
0347  * @offset: offset of the BO
0348  * @size: size of the BO
0349  * @domain: where to place it
0350  * @bo_ptr:  used to initialize BOs in structures
0351  * @cpu_addr: optional CPU address mapping
0352  *
0353  * Creates a kernel BO at a specific offset in the address space of the domain.
0354  *
0355  * Returns:
0356  * 0 on success, negative error code otherwise.
0357  */
0358 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
0359                    uint64_t offset, uint64_t size, uint32_t domain,
0360                    struct amdgpu_bo **bo_ptr, void **cpu_addr)
0361 {
0362     struct ttm_operation_ctx ctx = { false, false };
0363     unsigned int i;
0364     int r;
0365 
0366     offset &= PAGE_MASK;
0367     size = ALIGN(size, PAGE_SIZE);
0368 
0369     r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, domain, bo_ptr,
0370                       NULL, cpu_addr);
0371     if (r)
0372         return r;
0373 
0374     if ((*bo_ptr) == NULL)
0375         return 0;
0376 
0377     /*
0378      * Remove the original mem node and create a new one at the request
0379      * position.
0380      */
0381     if (cpu_addr)
0382         amdgpu_bo_kunmap(*bo_ptr);
0383 
0384     ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
0385 
0386     for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
0387         (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
0388         (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
0389     }
0390     r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
0391                  &(*bo_ptr)->tbo.resource, &ctx);
0392     if (r)
0393         goto error;
0394 
0395     if (cpu_addr) {
0396         r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
0397         if (r)
0398             goto error;
0399     }
0400 
0401     amdgpu_bo_unreserve(*bo_ptr);
0402     return 0;
0403 
0404 error:
0405     amdgpu_bo_unreserve(*bo_ptr);
0406     amdgpu_bo_unref(bo_ptr);
0407     return r;
0408 }
0409 
0410 /**
0411  * amdgpu_bo_free_kernel - free BO for kernel use
0412  *
0413  * @bo: amdgpu BO to free
0414  * @gpu_addr: pointer to where the BO's GPU memory space address was stored
0415  * @cpu_addr: pointer to where the BO's CPU memory space address was stored
0416  *
0417  * unmaps and unpin a BO for kernel internal use.
0418  */
0419 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
0420                void **cpu_addr)
0421 {
0422     if (*bo == NULL)
0423         return;
0424 
0425     if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
0426         if (cpu_addr)
0427             amdgpu_bo_kunmap(*bo);
0428 
0429         amdgpu_bo_unpin(*bo);
0430         amdgpu_bo_unreserve(*bo);
0431     }
0432     amdgpu_bo_unref(bo);
0433 
0434     if (gpu_addr)
0435         *gpu_addr = 0;
0436 
0437     if (cpu_addr)
0438         *cpu_addr = NULL;
0439 }
0440 
0441 /* Validate bo size is bit bigger then the request domain */
0442 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
0443                       unsigned long size, u32 domain)
0444 {
0445     struct ttm_resource_manager *man = NULL;
0446 
0447     /*
0448      * If GTT is part of requested domains the check must succeed to
0449      * allow fall back to GTT
0450      */
0451     if (domain & AMDGPU_GEM_DOMAIN_GTT) {
0452         man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
0453 
0454         if (size < man->size)
0455             return true;
0456         else
0457             goto fail;
0458     }
0459 
0460     if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
0461         man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
0462 
0463         if (size < man->size)
0464             return true;
0465         else
0466             goto fail;
0467     }
0468 
0469 
0470     /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */
0471     return true;
0472 
0473 fail:
0474     DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,
0475           man->size);
0476     return false;
0477 }
0478 
0479 bool amdgpu_bo_support_uswc(u64 bo_flags)
0480 {
0481 
0482 #ifdef CONFIG_X86_32
0483     /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
0484      * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
0485      */
0486     return false;
0487 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
0488     /* Don't try to enable write-combining when it can't work, or things
0489      * may be slow
0490      * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
0491      */
0492 
0493 #ifndef CONFIG_COMPILE_TEST
0494 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
0495      thanks to write-combining
0496 #endif
0497 
0498     if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
0499         DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
0500                   "better performance thanks to write-combining\n");
0501     return false;
0502 #else
0503     /* For architectures that don't support WC memory,
0504      * mask out the WC flag from the BO
0505      */
0506     if (!drm_arch_can_wc_memory())
0507         return false;
0508 
0509     return true;
0510 #endif
0511 }
0512 
0513 /**
0514  * amdgpu_bo_create - create an &amdgpu_bo buffer object
0515  * @adev: amdgpu device object
0516  * @bp: parameters to be used for the buffer object
0517  * @bo_ptr: pointer to the buffer object pointer
0518  *
0519  * Creates an &amdgpu_bo buffer object.
0520  *
0521  * Returns:
0522  * 0 for success or a negative error code on failure.
0523  */
0524 int amdgpu_bo_create(struct amdgpu_device *adev,
0525                    struct amdgpu_bo_param *bp,
0526                    struct amdgpu_bo **bo_ptr)
0527 {
0528     struct ttm_operation_ctx ctx = {
0529         .interruptible = (bp->type != ttm_bo_type_kernel),
0530         .no_wait_gpu = bp->no_wait_gpu,
0531         /* We opt to avoid OOM on system pages allocations */
0532         .gfp_retry_mayfail = true,
0533         .allow_res_evict = bp->type != ttm_bo_type_kernel,
0534         .resv = bp->resv
0535     };
0536     struct amdgpu_bo *bo;
0537     unsigned long page_align, size = bp->size;
0538     int r;
0539 
0540     /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
0541     if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
0542         /* GWS and OA don't need any alignment. */
0543         page_align = bp->byte_align;
0544         size <<= PAGE_SHIFT;
0545     } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) {
0546         /* Both size and alignment must be a multiple of 4. */
0547         page_align = ALIGN(bp->byte_align, 4);
0548         size = ALIGN(size, 4) << PAGE_SHIFT;
0549     } else {
0550         /* Memory should be aligned at least to a page size. */
0551         page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
0552         size = ALIGN(size, PAGE_SIZE);
0553     }
0554 
0555     if (!amdgpu_bo_validate_size(adev, size, bp->domain))
0556         return -ENOMEM;
0557 
0558     BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo));
0559 
0560     *bo_ptr = NULL;
0561     bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);
0562     if (bo == NULL)
0563         return -ENOMEM;
0564     drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);
0565     bo->vm_bo = NULL;
0566     bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
0567         bp->domain;
0568     bo->allowed_domains = bo->preferred_domains;
0569     if (bp->type != ttm_bo_type_kernel &&
0570         !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
0571         bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
0572         bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
0573 
0574     bo->flags = bp->flags;
0575 
0576     if (!amdgpu_bo_support_uswc(bo->flags))
0577         bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
0578 
0579     if (adev->ras_enabled)
0580         bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
0581 
0582     bo->tbo.bdev = &adev->mman.bdev;
0583     if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
0584               AMDGPU_GEM_DOMAIN_GDS))
0585         amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
0586     else
0587         amdgpu_bo_placement_from_domain(bo, bp->domain);
0588     if (bp->type == ttm_bo_type_kernel)
0589         bo->tbo.priority = 1;
0590 
0591     if (!bp->destroy)
0592         bp->destroy = &amdgpu_bo_destroy;
0593 
0594     r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,
0595                  &bo->placement, page_align, &ctx,  NULL,
0596                  bp->resv, bp->destroy);
0597     if (unlikely(r != 0))
0598         return r;
0599 
0600     if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
0601         bo->tbo.resource->mem_type == TTM_PL_VRAM &&
0602         bo->tbo.resource->start < adev->gmc.visible_vram_size >> PAGE_SHIFT)
0603         amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
0604                          ctx.bytes_moved);
0605     else
0606         amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
0607 
0608     if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
0609         bo->tbo.resource->mem_type == TTM_PL_VRAM) {
0610         struct dma_fence *fence;
0611 
0612         r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence);
0613         if (unlikely(r))
0614             goto fail_unreserve;
0615 
0616         dma_resv_add_fence(bo->tbo.base.resv, fence,
0617                    DMA_RESV_USAGE_KERNEL);
0618         dma_fence_put(fence);
0619     }
0620     if (!bp->resv)
0621         amdgpu_bo_unreserve(bo);
0622     *bo_ptr = bo;
0623 
0624     trace_amdgpu_bo_create(bo);
0625 
0626     /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
0627     if (bp->type == ttm_bo_type_device)
0628         bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
0629 
0630     return 0;
0631 
0632 fail_unreserve:
0633     if (!bp->resv)
0634         dma_resv_unlock(bo->tbo.base.resv);
0635     amdgpu_bo_unref(&bo);
0636     return r;
0637 }
0638 
0639 /**
0640  * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
0641  * @adev: amdgpu device object
0642  * @bp: parameters to be used for the buffer object
0643  * @ubo_ptr: pointer to the buffer object pointer
0644  *
0645  * Create a BO to be used by user application;
0646  *
0647  * Returns:
0648  * 0 for success or a negative error code on failure.
0649  */
0650 
0651 int amdgpu_bo_create_user(struct amdgpu_device *adev,
0652               struct amdgpu_bo_param *bp,
0653               struct amdgpu_bo_user **ubo_ptr)
0654 {
0655     struct amdgpu_bo *bo_ptr;
0656     int r;
0657 
0658     bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
0659     bp->destroy = &amdgpu_bo_user_destroy;
0660     r = amdgpu_bo_create(adev, bp, &bo_ptr);
0661     if (r)
0662         return r;
0663 
0664     *ubo_ptr = to_amdgpu_bo_user(bo_ptr);
0665     return r;
0666 }
0667 
0668 /**
0669  * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
0670  * @adev: amdgpu device object
0671  * @bp: parameters to be used for the buffer object
0672  * @vmbo_ptr: pointer to the buffer object pointer
0673  *
0674  * Create a BO to be for GPUVM.
0675  *
0676  * Returns:
0677  * 0 for success or a negative error code on failure.
0678  */
0679 
0680 int amdgpu_bo_create_vm(struct amdgpu_device *adev,
0681             struct amdgpu_bo_param *bp,
0682             struct amdgpu_bo_vm **vmbo_ptr)
0683 {
0684     struct amdgpu_bo *bo_ptr;
0685     int r;
0686 
0687     /* bo_ptr_size will be determined by the caller and it depends on
0688      * num of amdgpu_vm_pt entries.
0689      */
0690     BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
0691     bp->destroy = &amdgpu_bo_vm_destroy;
0692     r = amdgpu_bo_create(adev, bp, &bo_ptr);
0693     if (r)
0694         return r;
0695 
0696     *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
0697     INIT_LIST_HEAD(&(*vmbo_ptr)->shadow_list);
0698     return r;
0699 }
0700 
0701 /**
0702  * amdgpu_bo_add_to_shadow_list - add a BO to the shadow list
0703  *
0704  * @vmbo: BO that will be inserted into the shadow list
0705  *
0706  * Insert a BO to the shadow list.
0707  */
0708 void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm *vmbo)
0709 {
0710     struct amdgpu_device *adev = amdgpu_ttm_adev(vmbo->bo.tbo.bdev);
0711 
0712     mutex_lock(&adev->shadow_list_lock);
0713     list_add_tail(&vmbo->shadow_list, &adev->shadow_list);
0714     mutex_unlock(&adev->shadow_list_lock);
0715 }
0716 
0717 /**
0718  * amdgpu_bo_restore_shadow - restore an &amdgpu_bo shadow
0719  *
0720  * @shadow: &amdgpu_bo shadow to be restored
0721  * @fence: dma_fence associated with the operation
0722  *
0723  * Copies a buffer object's shadow content back to the object.
0724  * This is used for recovering a buffer from its shadow in case of a gpu
0725  * reset where vram context may be lost.
0726  *
0727  * Returns:
0728  * 0 for success or a negative error code on failure.
0729  */
0730 int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence)
0731 
0732 {
0733     struct amdgpu_device *adev = amdgpu_ttm_adev(shadow->tbo.bdev);
0734     struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
0735     uint64_t shadow_addr, parent_addr;
0736 
0737     shadow_addr = amdgpu_bo_gpu_offset(shadow);
0738     parent_addr = amdgpu_bo_gpu_offset(shadow->parent);
0739 
0740     return amdgpu_copy_buffer(ring, shadow_addr, parent_addr,
0741                   amdgpu_bo_size(shadow), NULL, fence,
0742                   true, false, false);
0743 }
0744 
0745 /**
0746  * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
0747  * @bo: &amdgpu_bo buffer object to be mapped
0748  * @ptr: kernel virtual address to be returned
0749  *
0750  * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
0751  * amdgpu_bo_kptr() to get the kernel virtual address.
0752  *
0753  * Returns:
0754  * 0 for success or a negative error code on failure.
0755  */
0756 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
0757 {
0758     void *kptr;
0759     long r;
0760 
0761     if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
0762         return -EPERM;
0763 
0764     r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL,
0765                   false, MAX_SCHEDULE_TIMEOUT);
0766     if (r < 0)
0767         return r;
0768 
0769     kptr = amdgpu_bo_kptr(bo);
0770     if (kptr) {
0771         if (ptr)
0772             *ptr = kptr;
0773         return 0;
0774     }
0775 
0776     r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.resource->num_pages, &bo->kmap);
0777     if (r)
0778         return r;
0779 
0780     if (ptr)
0781         *ptr = amdgpu_bo_kptr(bo);
0782 
0783     return 0;
0784 }
0785 
0786 /**
0787  * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
0788  * @bo: &amdgpu_bo buffer object
0789  *
0790  * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
0791  *
0792  * Returns:
0793  * the virtual address of a buffer object area.
0794  */
0795 void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
0796 {
0797     bool is_iomem;
0798 
0799     return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
0800 }
0801 
0802 /**
0803  * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
0804  * @bo: &amdgpu_bo buffer object to be unmapped
0805  *
0806  * Unmaps a kernel map set up by amdgpu_bo_kmap().
0807  */
0808 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
0809 {
0810     if (bo->kmap.bo)
0811         ttm_bo_kunmap(&bo->kmap);
0812 }
0813 
0814 /**
0815  * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
0816  * @bo: &amdgpu_bo buffer object
0817  *
0818  * References the contained &ttm_buffer_object.
0819  *
0820  * Returns:
0821  * a refcounted pointer to the &amdgpu_bo buffer object.
0822  */
0823 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
0824 {
0825     if (bo == NULL)
0826         return NULL;
0827 
0828     ttm_bo_get(&bo->tbo);
0829     return bo;
0830 }
0831 
0832 /**
0833  * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
0834  * @bo: &amdgpu_bo buffer object
0835  *
0836  * Unreferences the contained &ttm_buffer_object and clear the pointer
0837  */
0838 void amdgpu_bo_unref(struct amdgpu_bo **bo)
0839 {
0840     struct ttm_buffer_object *tbo;
0841 
0842     if ((*bo) == NULL)
0843         return;
0844 
0845     tbo = &((*bo)->tbo);
0846     ttm_bo_put(tbo);
0847     *bo = NULL;
0848 }
0849 
0850 /**
0851  * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object
0852  * @bo: &amdgpu_bo buffer object to be pinned
0853  * @domain: domain to be pinned to
0854  * @min_offset: the start of requested address range
0855  * @max_offset: the end of requested address range
0856  *
0857  * Pins the buffer object according to requested domain and address range. If
0858  * the memory is unbound gart memory, binds the pages into gart table. Adjusts
0859  * pin_count and pin_size accordingly.
0860  *
0861  * Pinning means to lock pages in memory along with keeping them at a fixed
0862  * offset. It is required when a buffer can not be moved, for example, when
0863  * a display buffer is being scanned out.
0864  *
0865  * Compared with amdgpu_bo_pin(), this function gives more flexibility on
0866  * where to pin a buffer if there are specific restrictions on where a buffer
0867  * must be located.
0868  *
0869  * Returns:
0870  * 0 for success or a negative error code on failure.
0871  */
0872 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
0873                  u64 min_offset, u64 max_offset)
0874 {
0875     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
0876     struct ttm_operation_ctx ctx = { false, false };
0877     int r, i;
0878 
0879     if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
0880         return -EPERM;
0881 
0882     if (WARN_ON_ONCE(min_offset > max_offset))
0883         return -EINVAL;
0884 
0885     /* Check domain to be pinned to against preferred domains */
0886     if (bo->preferred_domains & domain)
0887         domain = bo->preferred_domains & domain;
0888 
0889     /* A shared bo cannot be migrated to VRAM */
0890     if (bo->tbo.base.import_attach) {
0891         if (domain & AMDGPU_GEM_DOMAIN_GTT)
0892             domain = AMDGPU_GEM_DOMAIN_GTT;
0893         else
0894             return -EINVAL;
0895     }
0896 
0897     if (bo->tbo.pin_count) {
0898         uint32_t mem_type = bo->tbo.resource->mem_type;
0899         uint32_t mem_flags = bo->tbo.resource->placement;
0900 
0901         if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
0902             return -EINVAL;
0903 
0904         if ((mem_type == TTM_PL_VRAM) &&
0905             (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&
0906             !(mem_flags & TTM_PL_FLAG_CONTIGUOUS))
0907             return -EINVAL;
0908 
0909         ttm_bo_pin(&bo->tbo);
0910 
0911         if (max_offset != 0) {
0912             u64 domain_start = amdgpu_ttm_domain_start(adev,
0913                                    mem_type);
0914             WARN_ON_ONCE(max_offset <
0915                      (amdgpu_bo_gpu_offset(bo) - domain_start));
0916         }
0917 
0918         return 0;
0919     }
0920 
0921     /* This assumes only APU display buffers are pinned with (VRAM|GTT).
0922      * See function amdgpu_display_supported_domains()
0923      */
0924     domain = amdgpu_bo_get_preferred_domain(adev, domain);
0925 
0926     if (bo->tbo.base.import_attach)
0927         dma_buf_pin(bo->tbo.base.import_attach);
0928 
0929     /* force to pin into visible video ram */
0930     if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
0931         bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
0932     amdgpu_bo_placement_from_domain(bo, domain);
0933     for (i = 0; i < bo->placement.num_placement; i++) {
0934         unsigned fpfn, lpfn;
0935 
0936         fpfn = min_offset >> PAGE_SHIFT;
0937         lpfn = max_offset >> PAGE_SHIFT;
0938 
0939         if (fpfn > bo->placements[i].fpfn)
0940             bo->placements[i].fpfn = fpfn;
0941         if (!bo->placements[i].lpfn ||
0942             (lpfn && lpfn < bo->placements[i].lpfn))
0943             bo->placements[i].lpfn = lpfn;
0944     }
0945 
0946     r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
0947     if (unlikely(r)) {
0948         dev_err(adev->dev, "%p pin failed\n", bo);
0949         goto error;
0950     }
0951 
0952     ttm_bo_pin(&bo->tbo);
0953 
0954     domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
0955     if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
0956         atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
0957         atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
0958                  &adev->visible_pin_size);
0959     } else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
0960         atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
0961     }
0962 
0963 error:
0964     return r;
0965 }
0966 
0967 /**
0968  * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
0969  * @bo: &amdgpu_bo buffer object to be pinned
0970  * @domain: domain to be pinned to
0971  *
0972  * A simple wrapper to amdgpu_bo_pin_restricted().
0973  * Provides a simpler API for buffers that do not have any strict restrictions
0974  * on where a buffer must be located.
0975  *
0976  * Returns:
0977  * 0 for success or a negative error code on failure.
0978  */
0979 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
0980 {
0981     bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
0982     return amdgpu_bo_pin_restricted(bo, domain, 0, 0);
0983 }
0984 
0985 /**
0986  * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
0987  * @bo: &amdgpu_bo buffer object to be unpinned
0988  *
0989  * Decreases the pin_count, and clears the flags if pin_count reaches 0.
0990  * Changes placement and pin size accordingly.
0991  *
0992  * Returns:
0993  * 0 for success or a negative error code on failure.
0994  */
0995 void amdgpu_bo_unpin(struct amdgpu_bo *bo)
0996 {
0997     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
0998 
0999     ttm_bo_unpin(&bo->tbo);
1000     if (bo->tbo.pin_count)
1001         return;
1002 
1003     if (bo->tbo.base.import_attach)
1004         dma_buf_unpin(bo->tbo.base.import_attach);
1005 
1006     if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1007         atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
1008         atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
1009                  &adev->visible_pin_size);
1010     } else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1011         atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
1012     }
1013 }
1014 
1015 static const char *amdgpu_vram_names[] = {
1016     "UNKNOWN",
1017     "GDDR1",
1018     "DDR2",
1019     "GDDR3",
1020     "GDDR4",
1021     "GDDR5",
1022     "HBM",
1023     "DDR3",
1024     "DDR4",
1025     "GDDR6",
1026     "DDR5",
1027     "LPDDR4",
1028     "LPDDR5"
1029 };
1030 
1031 /**
1032  * amdgpu_bo_init - initialize memory manager
1033  * @adev: amdgpu device object
1034  *
1035  * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1036  *
1037  * Returns:
1038  * 0 for success or a negative error code on failure.
1039  */
1040 int amdgpu_bo_init(struct amdgpu_device *adev)
1041 {
1042     /* On A+A platform, VRAM can be mapped as WB */
1043     if (!adev->gmc.xgmi.connected_to_cpu) {
1044         /* reserve PAT memory space to WC for VRAM */
1045         int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base,
1046                 adev->gmc.aper_size);
1047 
1048         if (r) {
1049             DRM_ERROR("Unable to set WC memtype for the aperture base\n");
1050             return r;
1051         }
1052 
1053         /* Add an MTRR for the VRAM */
1054         adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
1055                 adev->gmc.aper_size);
1056     }
1057 
1058     DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
1059          adev->gmc.mc_vram_size >> 20,
1060          (unsigned long long)adev->gmc.aper_size >> 20);
1061     DRM_INFO("RAM width %dbits %s\n",
1062          adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
1063     return amdgpu_ttm_init(adev);
1064 }
1065 
1066 /**
1067  * amdgpu_bo_fini - tear down memory manager
1068  * @adev: amdgpu device object
1069  *
1070  * Reverses amdgpu_bo_init() to tear down memory manager.
1071  */
1072 void amdgpu_bo_fini(struct amdgpu_device *adev)
1073 {
1074     int idx;
1075 
1076     amdgpu_ttm_fini(adev);
1077 
1078     if (drm_dev_enter(adev_to_drm(adev), &idx)) {
1079 
1080         if (!adev->gmc.xgmi.connected_to_cpu) {
1081             arch_phys_wc_del(adev->gmc.vram_mtrr);
1082             arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
1083         }
1084         drm_dev_exit(idx);
1085     }
1086 }
1087 
1088 /**
1089  * amdgpu_bo_set_tiling_flags - set tiling flags
1090  * @bo: &amdgpu_bo buffer object
1091  * @tiling_flags: new flags
1092  *
1093  * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1094  * kernel driver to set the tiling flags on a buffer.
1095  *
1096  * Returns:
1097  * 0 for success or a negative error code on failure.
1098  */
1099 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1100 {
1101     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1102     struct amdgpu_bo_user *ubo;
1103 
1104     BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1105     if (adev->family <= AMDGPU_FAMILY_CZ &&
1106         AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1107         return -EINVAL;
1108 
1109     ubo = to_amdgpu_bo_user(bo);
1110     ubo->tiling_flags = tiling_flags;
1111     return 0;
1112 }
1113 
1114 /**
1115  * amdgpu_bo_get_tiling_flags - get tiling flags
1116  * @bo: &amdgpu_bo buffer object
1117  * @tiling_flags: returned flags
1118  *
1119  * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1120  * set the tiling flags on a buffer.
1121  */
1122 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1123 {
1124     struct amdgpu_bo_user *ubo;
1125 
1126     BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1127     dma_resv_assert_held(bo->tbo.base.resv);
1128     ubo = to_amdgpu_bo_user(bo);
1129 
1130     if (tiling_flags)
1131         *tiling_flags = ubo->tiling_flags;
1132 }
1133 
1134 /**
1135  * amdgpu_bo_set_metadata - set metadata
1136  * @bo: &amdgpu_bo buffer object
1137  * @metadata: new metadata
1138  * @metadata_size: size of the new metadata
1139  * @flags: flags of the new metadata
1140  *
1141  * Sets buffer object's metadata, its size and flags.
1142  * Used via GEM ioctl.
1143  *
1144  * Returns:
1145  * 0 for success or a negative error code on failure.
1146  */
1147 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
1148                 uint32_t metadata_size, uint64_t flags)
1149 {
1150     struct amdgpu_bo_user *ubo;
1151     void *buffer;
1152 
1153     BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1154     ubo = to_amdgpu_bo_user(bo);
1155     if (!metadata_size) {
1156         if (ubo->metadata_size) {
1157             kfree(ubo->metadata);
1158             ubo->metadata = NULL;
1159             ubo->metadata_size = 0;
1160         }
1161         return 0;
1162     }
1163 
1164     if (metadata == NULL)
1165         return -EINVAL;
1166 
1167     buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1168     if (buffer == NULL)
1169         return -ENOMEM;
1170 
1171     kfree(ubo->metadata);
1172     ubo->metadata_flags = flags;
1173     ubo->metadata = buffer;
1174     ubo->metadata_size = metadata_size;
1175 
1176     return 0;
1177 }
1178 
1179 /**
1180  * amdgpu_bo_get_metadata - get metadata
1181  * @bo: &amdgpu_bo buffer object
1182  * @buffer: returned metadata
1183  * @buffer_size: size of the buffer
1184  * @metadata_size: size of the returned metadata
1185  * @flags: flags of the returned metadata
1186  *
1187  * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1188  * less than metadata_size.
1189  * Used via GEM ioctl.
1190  *
1191  * Returns:
1192  * 0 for success or a negative error code on failure.
1193  */
1194 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1195                size_t buffer_size, uint32_t *metadata_size,
1196                uint64_t *flags)
1197 {
1198     struct amdgpu_bo_user *ubo;
1199 
1200     if (!buffer && !metadata_size)
1201         return -EINVAL;
1202 
1203     BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1204     ubo = to_amdgpu_bo_user(bo);
1205     if (metadata_size)
1206         *metadata_size = ubo->metadata_size;
1207 
1208     if (buffer) {
1209         if (buffer_size < ubo->metadata_size)
1210             return -EINVAL;
1211 
1212         if (ubo->metadata_size)
1213             memcpy(buffer, ubo->metadata, ubo->metadata_size);
1214     }
1215 
1216     if (flags)
1217         *flags = ubo->metadata_flags;
1218 
1219     return 0;
1220 }
1221 
1222 /**
1223  * amdgpu_bo_move_notify - notification about a memory move
1224  * @bo: pointer to a buffer object
1225  * @evict: if this move is evicting the buffer from the graphics address space
1226  * @new_mem: new information of the bufer object
1227  *
1228  * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1229  * bookkeeping.
1230  * TTM driver callback which is called when ttm moves a buffer.
1231  */
1232 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1233                bool evict,
1234                struct ttm_resource *new_mem)
1235 {
1236     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1237     struct amdgpu_bo *abo;
1238     struct ttm_resource *old_mem = bo->resource;
1239 
1240     if (!amdgpu_bo_is_amdgpu_bo(bo))
1241         return;
1242 
1243     abo = ttm_to_amdgpu_bo(bo);
1244     amdgpu_vm_bo_invalidate(adev, abo, evict);
1245 
1246     amdgpu_bo_kunmap(abo);
1247 
1248     if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach &&
1249         bo->resource->mem_type != TTM_PL_SYSTEM)
1250         dma_buf_move_notify(abo->tbo.base.dma_buf);
1251 
1252     /* remember the eviction */
1253     if (evict)
1254         atomic64_inc(&adev->num_evictions);
1255 
1256     /* update statistics */
1257     if (!new_mem)
1258         return;
1259 
1260     /* move_notify is called before move happens */
1261     trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
1262 }
1263 
1264 void amdgpu_bo_get_memory(struct amdgpu_bo *bo, uint64_t *vram_mem,
1265                 uint64_t *gtt_mem, uint64_t *cpu_mem)
1266 {
1267     unsigned int domain;
1268 
1269     domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
1270     switch (domain) {
1271     case AMDGPU_GEM_DOMAIN_VRAM:
1272         *vram_mem += amdgpu_bo_size(bo);
1273         break;
1274     case AMDGPU_GEM_DOMAIN_GTT:
1275         *gtt_mem += amdgpu_bo_size(bo);
1276         break;
1277     case AMDGPU_GEM_DOMAIN_CPU:
1278     default:
1279         *cpu_mem += amdgpu_bo_size(bo);
1280         break;
1281     }
1282 }
1283 
1284 /**
1285  * amdgpu_bo_release_notify - notification about a BO being released
1286  * @bo: pointer to a buffer object
1287  *
1288  * Wipes VRAM buffers whose contents should not be leaked before the
1289  * memory is released.
1290  */
1291 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
1292 {
1293     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1294     struct dma_fence *fence = NULL;
1295     struct amdgpu_bo *abo;
1296     int r;
1297 
1298     if (!amdgpu_bo_is_amdgpu_bo(bo))
1299         return;
1300 
1301     abo = ttm_to_amdgpu_bo(bo);
1302 
1303     if (abo->kfd_bo)
1304         amdgpu_amdkfd_release_notify(abo);
1305 
1306     /* We only remove the fence if the resv has individualized. */
1307     WARN_ON_ONCE(bo->type == ttm_bo_type_kernel
1308             && bo->base.resv != &bo->base._resv);
1309     if (bo->base.resv == &bo->base._resv)
1310         amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo);
1311 
1312     if (bo->resource->mem_type != TTM_PL_VRAM ||
1313         !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) ||
1314         adev->in_suspend || adev->shutdown)
1315         return;
1316 
1317     if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv)))
1318         return;
1319 
1320     r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence);
1321     if (!WARN_ON(r)) {
1322         amdgpu_bo_fence(abo, fence, false);
1323         dma_fence_put(fence);
1324     }
1325 
1326     dma_resv_unlock(bo->base.resv);
1327 }
1328 
1329 /**
1330  * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1331  * @bo: pointer to a buffer object
1332  *
1333  * Notifies the driver we are taking a fault on this BO and have reserved it,
1334  * also performs bookkeeping.
1335  * TTM driver callback for dealing with vm faults.
1336  *
1337  * Returns:
1338  * 0 for success or a negative error code on failure.
1339  */
1340 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1341 {
1342     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1343     struct ttm_operation_ctx ctx = { false, false };
1344     struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1345     unsigned long offset;
1346     int r;
1347 
1348     /* Remember that this BO was accessed by the CPU */
1349     abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1350 
1351     if (bo->resource->mem_type != TTM_PL_VRAM)
1352         return 0;
1353 
1354     offset = bo->resource->start << PAGE_SHIFT;
1355     if ((offset + bo->base.size) <= adev->gmc.visible_vram_size)
1356         return 0;
1357 
1358     /* Can't move a pinned BO to visible VRAM */
1359     if (abo->tbo.pin_count > 0)
1360         return VM_FAULT_SIGBUS;
1361 
1362     /* hurrah the memory is not visible ! */
1363     atomic64_inc(&adev->num_vram_cpu_page_faults);
1364     amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1365                     AMDGPU_GEM_DOMAIN_GTT);
1366 
1367     /* Avoid costly evictions; only set GTT as a busy placement */
1368     abo->placement.num_busy_placement = 1;
1369     abo->placement.busy_placement = &abo->placements[1];
1370 
1371     r = ttm_bo_validate(bo, &abo->placement, &ctx);
1372     if (unlikely(r == -EBUSY || r == -ERESTARTSYS))
1373         return VM_FAULT_NOPAGE;
1374     else if (unlikely(r))
1375         return VM_FAULT_SIGBUS;
1376 
1377     offset = bo->resource->start << PAGE_SHIFT;
1378     /* this should never happen */
1379     if (bo->resource->mem_type == TTM_PL_VRAM &&
1380         (offset + bo->base.size) > adev->gmc.visible_vram_size)
1381         return VM_FAULT_SIGBUS;
1382 
1383     ttm_bo_move_to_lru_tail_unlocked(bo);
1384     return 0;
1385 }
1386 
1387 /**
1388  * amdgpu_bo_fence - add fence to buffer object
1389  *
1390  * @bo: buffer object in question
1391  * @fence: fence to add
1392  * @shared: true if fence should be added shared
1393  *
1394  */
1395 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1396              bool shared)
1397 {
1398     struct dma_resv *resv = bo->tbo.base.resv;
1399     int r;
1400 
1401     r = dma_resv_reserve_fences(resv, 1);
1402     if (r) {
1403         /* As last resort on OOM we block for the fence */
1404         dma_fence_wait(fence, false);
1405         return;
1406     }
1407 
1408     dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ :
1409                DMA_RESV_USAGE_WRITE);
1410 }
1411 
1412 /**
1413  * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1414  *
1415  * @adev: amdgpu device pointer
1416  * @resv: reservation object to sync to
1417  * @sync_mode: synchronization mode
1418  * @owner: fence owner
1419  * @intr: Whether the wait is interruptible
1420  *
1421  * Extract the fences from the reservation object and waits for them to finish.
1422  *
1423  * Returns:
1424  * 0 on success, errno otherwise.
1425  */
1426 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
1427                  enum amdgpu_sync_mode sync_mode, void *owner,
1428                  bool intr)
1429 {
1430     struct amdgpu_sync sync;
1431     int r;
1432 
1433     amdgpu_sync_create(&sync);
1434     amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner);
1435     r = amdgpu_sync_wait(&sync, intr);
1436     amdgpu_sync_free(&sync);
1437     return r;
1438 }
1439 
1440 /**
1441  * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1442  * @bo: buffer object to wait for
1443  * @owner: fence owner
1444  * @intr: Whether the wait is interruptible
1445  *
1446  * Wrapper to wait for fences in a BO.
1447  * Returns:
1448  * 0 on success, errno otherwise.
1449  */
1450 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)
1451 {
1452     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1453 
1454     return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,
1455                     AMDGPU_SYNC_NE_OWNER, owner, intr);
1456 }
1457 
1458 /**
1459  * amdgpu_bo_gpu_offset - return GPU offset of bo
1460  * @bo: amdgpu object for which we query the offset
1461  *
1462  * Note: object should either be pinned or reserved when calling this
1463  * function, it might be useful to add check for this for debugging.
1464  *
1465  * Returns:
1466  * current GPU offset of the object.
1467  */
1468 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1469 {
1470     WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);
1471     WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&
1472              !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);
1473     WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);
1474     WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&
1475              !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1476 
1477     return amdgpu_bo_gpu_offset_no_check(bo);
1478 }
1479 
1480 /**
1481  * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1482  * @bo: amdgpu object for which we query the offset
1483  *
1484  * Returns:
1485  * current GPU offset of the object without raising warnings.
1486  */
1487 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)
1488 {
1489     struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1490     uint64_t offset;
1491 
1492     offset = (bo->tbo.resource->start << PAGE_SHIFT) +
1493          amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);
1494 
1495     return amdgpu_gmc_sign_extend(offset);
1496 }
1497 
1498 /**
1499  * amdgpu_bo_get_preferred_domain - get preferred domain
1500  * @adev: amdgpu device object
1501  * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1502  *
1503  * Returns:
1504  * Which of the allowed domains is preferred for allocating the BO.
1505  */
1506 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
1507                         uint32_t domain)
1508 {
1509     if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) {
1510         domain = AMDGPU_GEM_DOMAIN_VRAM;
1511         if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1512             domain = AMDGPU_GEM_DOMAIN_GTT;
1513     }
1514     return domain;
1515 }
1516 
1517 #if defined(CONFIG_DEBUG_FS)
1518 #define amdgpu_bo_print_flag(m, bo, flag)               \
1519     do {                            \
1520         if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \
1521             seq_printf((m), " " #flag);     \
1522         }                       \
1523     } while (0)
1524 
1525 /**
1526  * amdgpu_bo_print_info - print BO info in debugfs file
1527  *
1528  * @id: Index or Id of the BO
1529  * @bo: Requested BO for printing info
1530  * @m: debugfs file
1531  *
1532  * Print BO information in debugfs file
1533  *
1534  * Returns:
1535  * Size of the BO in bytes.
1536  */
1537 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
1538 {
1539     struct dma_buf_attachment *attachment;
1540     struct dma_buf *dma_buf;
1541     unsigned int domain;
1542     const char *placement;
1543     unsigned int pin_count;
1544     u64 size;
1545 
1546     domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
1547     switch (domain) {
1548     case AMDGPU_GEM_DOMAIN_VRAM:
1549         placement = "VRAM";
1550         break;
1551     case AMDGPU_GEM_DOMAIN_GTT:
1552         placement = " GTT";
1553         break;
1554     case AMDGPU_GEM_DOMAIN_CPU:
1555     default:
1556         placement = " CPU";
1557         break;
1558     }
1559 
1560     size = amdgpu_bo_size(bo);
1561     seq_printf(m, "\t\t0x%08x: %12lld byte %s",
1562             id, size, placement);
1563 
1564     pin_count = READ_ONCE(bo->tbo.pin_count);
1565     if (pin_count)
1566         seq_printf(m, " pin count %d", pin_count);
1567 
1568     dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
1569     attachment = READ_ONCE(bo->tbo.base.import_attach);
1570 
1571     if (attachment)
1572         seq_printf(m, " imported from %p", dma_buf);
1573     else if (dma_buf)
1574         seq_printf(m, " exported as %p", dma_buf);
1575 
1576     amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
1577     amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1578     amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1579     amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1580     amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1581     amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1582     amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);
1583 
1584     seq_puts(m, "\n");
1585 
1586     return size;
1587 }
1588 #endif
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!