drm/i915/i915_ttm_buddy_manager.c

0001 // SPDX-License-Identifier: MIT
0002 /*
0003  * Copyright © 2021 Intel Corporation
0004  */
0005
0006 #include <linux/slab.h>
0007
0008 #include <drm/ttm/ttm_bo_driver.h>
0009 #include <drm/ttm/ttm_placement.h>
0010
0011 #include <drm/drm_buddy.h>
0012
0013 #include "i915_ttm_buddy_manager.h"
0014
0015 #include "i915_gem.h"
0016
0017 struct i915_ttm_buddy_manager {
0018     struct ttm_resource_manager manager;
0019     struct drm_buddy mm;
0020     struct list_head reserved;
0021     struct mutex lock;
0022     unsigned long visible_size;
0023     unsigned long visible_avail;
0024     unsigned long visible_reserved;
0025     u64 default_page_size;
0026 };
0027
0028 static struct i915_ttm_buddy_manager *
0029 to_buddy_manager(struct ttm_resource_manager *man)
0030 {
0031     return container_of(man, struct i915_ttm_buddy_manager, manager);
0032 }
0033
0034 static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man,
0035                     struct ttm_buffer_object *bo,
0036                     const struct ttm_place *place,
0037                     struct ttm_resource **res)
0038 {
0039     struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
0040     struct i915_ttm_buddy_resource *bman_res;
0041     struct drm_buddy *mm = &bman->mm;
0042     unsigned long n_pages, lpfn;
0043     u64 min_page_size;
0044     u64 size;
0045     int err;
0046
0047     lpfn = place->lpfn;
0048     if (!lpfn)
0049         lpfn = man->size;
0050
0051     bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL);
0052     if (!bman_res)
0053         return -ENOMEM;
0054
0055     ttm_resource_init(bo, place, &bman_res->base);
0056     INIT_LIST_HEAD(&bman_res->blocks);
0057     bman_res->mm = mm;
0058
0059     if (place->flags & TTM_PL_FLAG_TOPDOWN)
0060         bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
0061
0062     if (place->fpfn || lpfn != man->size)
0063         bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION;
0064
0065     GEM_BUG_ON(!bman_res->base.num_pages);
0066     size = bman_res->base.num_pages << PAGE_SHIFT;
0067
0068     min_page_size = bman->default_page_size;
0069     if (bo->page_alignment)
0070         min_page_size = bo->page_alignment << PAGE_SHIFT;
0071
0072     GEM_BUG_ON(min_page_size < mm->chunk_size);
0073     GEM_BUG_ON(!IS_ALIGNED(size, min_page_size));
0074
0075     if (place->fpfn + bman_res->base.num_pages != place->lpfn &&
0076         place->flags & TTM_PL_FLAG_CONTIGUOUS) {
0077         unsigned long pages;
0078
0079         size = roundup_pow_of_two(size);
0080         min_page_size = size;
0081
0082         pages = size >> ilog2(mm->chunk_size);
0083         if (pages > lpfn)
0084             lpfn = pages;
0085     }
0086
0087     if (size > lpfn << PAGE_SHIFT) {
0088         err = -E2BIG;
0089         goto err_free_res;
0090     }
0091
0092     n_pages = size >> ilog2(mm->chunk_size);
0093
0094     mutex_lock(&bman->lock);
0095     if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) {
0096         mutex_unlock(&bman->lock);
0097         err = -ENOSPC;
0098         goto err_free_res;
0099     }
0100
0101     err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT,
0102                      (u64)lpfn << PAGE_SHIFT,
0103                      (u64)n_pages << PAGE_SHIFT,
0104                      min_page_size,
0105                      &bman_res->blocks,
0106                      bman_res->flags);
0107     if (unlikely(err))
0108         goto err_free_blocks;
0109
0110     if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
0111         u64 original_size = (u64)bman_res->base.num_pages << PAGE_SHIFT;
0112
0113         drm_buddy_block_trim(mm,
0114                      original_size,
0115                      &bman_res->blocks);
0116     }
0117
0118     if (lpfn <= bman->visible_size) {
0119         bman_res->used_visible_size = bman_res->base.num_pages;
0120     } else {
0121         struct drm_buddy_block *block;
0122
0123         list_for_each_entry(block, &bman_res->blocks, link) {
0124             unsigned long start =
0125                 drm_buddy_block_offset(block) >> PAGE_SHIFT;
0126
0127             if (start < bman->visible_size) {
0128                 unsigned long end = start +
0129                     (drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
0130
0131                 bman_res->used_visible_size +=
0132                     min(end, bman->visible_size) - start;
0133             }
0134         }
0135     }
0136
0137     if (bman_res->used_visible_size)
0138         bman->visible_avail -= bman_res->used_visible_size;
0139
0140     mutex_unlock(&bman->lock);
0141
0142     if (place->lpfn - place->fpfn == n_pages)
0143         bman_res->base.start = place->fpfn;
0144     else if (lpfn <= bman->visible_size)
0145         bman_res->base.start = 0;
0146     else
0147         bman_res->base.start = bman->visible_size;
0148
0149     *res = &bman_res->base;
0150     return 0;
0151
0152 err_free_blocks:
0153     drm_buddy_free_list(mm, &bman_res->blocks);
0154     mutex_unlock(&bman->lock);
0155 err_free_res:
0156     ttm_resource_fini(man, &bman_res->base);
0157     kfree(bman_res);
0158     return err;
0159 }
0160
0161 static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man,
0162                     struct ttm_resource *res)
0163 {
0164     struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
0165     struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
0166
0167     mutex_lock(&bman->lock);
0168     drm_buddy_free_list(&bman->mm, &bman_res->blocks);
0169     bman->visible_avail += bman_res->used_visible_size;
0170     mutex_unlock(&bman->lock);
0171
0172     ttm_resource_fini(man, res);
0173     kfree(bman_res);
0174 }
0175
0176 static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man,
0177                      struct drm_printer *printer)
0178 {
0179     struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
0180     struct drm_buddy_block *block;
0181
0182     mutex_lock(&bman->lock);
0183     drm_printf(printer, "default_page_size: %lluKiB\n",
0184            bman->default_page_size >> 10);
0185     drm_printf(printer, "visible_avail: %lluMiB\n",
0186            (u64)bman->visible_avail << PAGE_SHIFT >> 20);
0187     drm_printf(printer, "visible_size: %lluMiB\n",
0188            (u64)bman->visible_size << PAGE_SHIFT >> 20);
0189     drm_printf(printer, "visible_reserved: %lluMiB\n",
0190            (u64)bman->visible_reserved << PAGE_SHIFT >> 20);
0191
0192     drm_buddy_print(&bman->mm, printer);
0193
0194     drm_printf(printer, "reserved:\n");
0195     list_for_each_entry(block, &bman->reserved, link)
0196         drm_buddy_block_print(&bman->mm, block, printer);
0197     mutex_unlock(&bman->lock);
0198 }
0199
0200 static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = {
0201     .alloc = i915_ttm_buddy_man_alloc,
0202     .free = i915_ttm_buddy_man_free,
0203     .debug = i915_ttm_buddy_man_debug,
0204 };
0205
0206 /**
0207  * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager
0208  * @bdev: The ttm device
0209  * @type: Memory type we want to manage
0210  * @use_tt: Set use_tt for the manager
0211  * @size: The size in bytes to manage
0212  * @visible_size: The CPU visible size in bytes to manage
0213  * @default_page_size: The default minimum page size in bytes for allocations,
0214  * this must be at least as large as @chunk_size, and can be overridden by
0215  * setting the BO page_alignment, to be larger or smaller as needed.
0216  * @chunk_size: The minimum page size in bytes for our allocations i.e
0217  * order-zero
0218  *
0219  * Note that the starting address is assumed to be zero here, since this
0220  * simplifies keeping the property where allocated blocks having natural
0221  * power-of-two alignment. So long as the real starting address is some large
0222  * power-of-two, or naturally start from zero, then this should be fine.  Also
0223  * the &i915_ttm_buddy_man_reserve interface can be used to preserve alignment
0224  * if say there is some unusable range from the start of the region. We can
0225  * revisit this in the future and make the interface accept an actual starting
0226  * offset and let it take care of the rest.
0227  *
0228  * Note that if the @size is not aligned to the @chunk_size then we perform the
0229  * required rounding to get the usable size. The final size in pages can be
0230  * taken from &ttm_resource_manager.size.
0231  *
0232  * Return: 0 on success, negative error code on failure.
0233  */
0234 int i915_ttm_buddy_man_init(struct ttm_device *bdev,
0235                 unsigned int type, bool use_tt,
0236                 u64 size, u64 visible_size, u64 default_page_size,
0237                 u64 chunk_size)
0238 {
0239     struct ttm_resource_manager *man;
0240     struct i915_ttm_buddy_manager *bman;
0241     int err;
0242
0243     bman = kzalloc(sizeof(*bman), GFP_KERNEL);
0244     if (!bman)
0245         return -ENOMEM;
0246
0247     err = drm_buddy_init(&bman->mm, size, chunk_size);
0248     if (err)
0249         goto err_free_bman;
0250
0251     mutex_init(&bman->lock);
0252     INIT_LIST_HEAD(&bman->reserved);
0253     GEM_BUG_ON(default_page_size < chunk_size);
0254     bman->default_page_size = default_page_size;
0255     bman->visible_size = visible_size >> PAGE_SHIFT;
0256     bman->visible_avail = bman->visible_size;
0257
0258     man = &bman->manager;
0259     man->use_tt = use_tt;
0260     man->func = &i915_ttm_buddy_manager_func;
0261     ttm_resource_manager_init(man, bdev, bman->mm.size >> PAGE_SHIFT);
0262
0263     ttm_resource_manager_set_used(man, true);
0264     ttm_set_driver_manager(bdev, type, man);
0265
0266     return 0;
0267
0268 err_free_bman:
0269     kfree(bman);
0270     return err;
0271 }
0272
0273 /**
0274  * i915_ttm_buddy_man_fini - Destroy the buddy allocator ttm manager
0275  * @bdev: The ttm device
0276  * @type: Memory type we want to manage
0277  *
0278  * Note that if we reserved anything with &i915_ttm_buddy_man_reserve, this will
0279  * also be freed for us here.
0280  *
0281  * Return: 0 on success, negative error code on failure.
0282  */
0283 int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type)
0284 {
0285     struct ttm_resource_manager *man = ttm_manager_type(bdev, type);
0286     struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
0287     struct drm_buddy *mm = &bman->mm;
0288     int ret;
0289
0290     ttm_resource_manager_set_used(man, false);
0291
0292     ret = ttm_resource_manager_evict_all(bdev, man);
0293     if (ret)
0294         return ret;
0295
0296     ttm_set_driver_manager(bdev, type, NULL);
0297
0298     mutex_lock(&bman->lock);
0299     drm_buddy_free_list(mm, &bman->reserved);
0300     drm_buddy_fini(mm);
0301     bman->visible_avail += bman->visible_reserved;
0302     WARN_ON_ONCE(bman->visible_avail != bman->visible_size);
0303     mutex_unlock(&bman->lock);
0304
0305     ttm_resource_manager_cleanup(man);
0306     kfree(bman);
0307
0308     return 0;
0309 }
0310
0311 /**
0312  * i915_ttm_buddy_man_reserve - Reserve address range
0313  * @man: The buddy allocator ttm manager
0314  * @start: The offset in bytes, where the region start is assumed to be zero
0315  * @size: The size in bytes
0316  *
0317  * Note that the starting address for the region is always assumed to be zero.
0318  *
0319  * Return: 0 on success, negative error code on failure.
0320  */
0321 int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man,
0322                    u64 start, u64 size)
0323 {
0324     struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
0325     struct drm_buddy *mm = &bman->mm;
0326     unsigned long fpfn = start >> PAGE_SHIFT;
0327     unsigned long flags = 0;
0328     int ret;
0329
0330     flags |= DRM_BUDDY_RANGE_ALLOCATION;
0331
0332     mutex_lock(&bman->lock);
0333     ret = drm_buddy_alloc_blocks(mm, start,
0334                      start + size,
0335                      size, mm->chunk_size,
0336                      &bman->reserved,
0337                      flags);
0338
0339     if (fpfn < bman->visible_size) {
0340         unsigned long lpfn = fpfn + (size >> PAGE_SHIFT);
0341         unsigned long visible = min(lpfn, bman->visible_size) - fpfn;
0342
0343         bman->visible_reserved += visible;
0344         bman->visible_avail -= visible;
0345     }
0346     mutex_unlock(&bman->lock);
0347
0348     return ret;
0349 }
0350
0351 /**
0352  * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion
0353  * in pages.
0354  * @man: The buddy allocator ttm manager
0355  */
0356 u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man)
0357 {
0358     struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
0359
0360     return bman->visible_size;
0361 }
0362
0363 /**
0364  * i915_ttm_buddy_man_avail - Query the avail tracking for the manager.
0365  *
0366  * @man: The buddy allocator ttm manager
0367  * @avail: The total available memory in pages for the entire manager.
0368  * @visible_avail: The total available memory in pages for the CPU visible
0369  * portion. Note that this will always give the same value as @avail on
0370  * configurations that don't have a small BAR.
0371  */
0372 void i915_ttm_buddy_man_avail(struct ttm_resource_manager *man,
0373                   u64 *avail, u64 *visible_avail)
0374 {
0375     struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
0376
0377     mutex_lock(&bman->lock);
0378     *avail = bman->mm.avail >> PAGE_SHIFT;
0379     *visible_avail = bman->visible_avail;
0380     mutex_unlock(&bman->lock);
0381 }
0382
0383 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
0384 void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man,
0385                        u64 size)
0386 {
0387     struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
0388
0389     bman->visible_size = size;
0390 }
0391 #endif