drm/radeon/radeon_gart.c

0001 /*
0002  * Copyright 2008 Advanced Micro Devices, Inc.
0003  * Copyright 2008 Red Hat Inc.
0004  * Copyright 2009 Jerome Glisse.
0005  *
0006  * Permission is hereby granted, free of charge, to any person obtaining a
0007  * copy of this software and associated documentation files (the "Software"),
0008  * to deal in the Software without restriction, including without limitation
0009  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0010  * and/or sell copies of the Software, and to permit persons to whom the
0011  * Software is furnished to do so, subject to the following conditions:
0012  *
0013  * The above copyright notice and this permission notice shall be included in
0014  * all copies or substantial portions of the Software.
0015  *
0016  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0017  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0018  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0019  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0020  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0021  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0022  * OTHER DEALINGS IN THE SOFTWARE.
0023  *
0024  * Authors: Dave Airlie
0025  *          Alex Deucher
0026  *          Jerome Glisse
0027  */
0028
0029 #include <linux/pci.h>
0030 #include <linux/vmalloc.h>
0031
0032 #include <drm/radeon_drm.h>
0033 #ifdef CONFIG_X86
0034 #include <asm/set_memory.h>
0035 #endif
0036 #include "radeon.h"
0037
0038 /*
0039  * GART
0040  * The GART (Graphics Aperture Remapping Table) is an aperture
0041  * in the GPU's address space.  System pages can be mapped into
0042  * the aperture and look like contiguous pages from the GPU's
0043  * perspective.  A page table maps the pages in the aperture
0044  * to the actual backing pages in system memory.
0045  *
0046  * Radeon GPUs support both an internal GART, as described above,
0047  * and AGP.  AGP works similarly, but the GART table is configured
0048  * and maintained by the northbridge rather than the driver.
0049  * Radeon hw has a separate AGP aperture that is programmed to
0050  * point to the AGP aperture provided by the northbridge and the
0051  * requests are passed through to the northbridge aperture.
0052  * Both AGP and internal GART can be used at the same time, however
0053  * that is not currently supported by the driver.
0054  *
0055  * This file handles the common internal GART management.
0056  */
0057
0058 /*
0059  * Common GART table functions.
0060  */
0061 /**
0062  * radeon_gart_table_ram_alloc - allocate system ram for gart page table
0063  *
0064  * @rdev: radeon_device pointer
0065  *
0066  * Allocate system memory for GART page table
0067  * (r1xx-r3xx, non-pcie r4xx, rs400).  These asics require the
0068  * gart table to be in system memory.
0069  * Returns 0 for success, -ENOMEM for failure.
0070  */
0071 int radeon_gart_table_ram_alloc(struct radeon_device *rdev)
0072 {
0073     void *ptr;
0074
0075     ptr = dma_alloc_coherent(&rdev->pdev->dev, rdev->gart.table_size,
0076                  &rdev->gart.table_addr, GFP_KERNEL);
0077     if (ptr == NULL) {
0078         return -ENOMEM;
0079     }
0080 #ifdef CONFIG_X86
0081     if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
0082         rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
0083         set_memory_uc((unsigned long)ptr,
0084                   rdev->gart.table_size >> PAGE_SHIFT);
0085     }
0086 #endif
0087     rdev->gart.ptr = ptr;
0088     return 0;
0089 }
0090
0091 /**
0092  * radeon_gart_table_ram_free - free system ram for gart page table
0093  *
0094  * @rdev: radeon_device pointer
0095  *
0096  * Free system memory for GART page table
0097  * (r1xx-r3xx, non-pcie r4xx, rs400).  These asics require the
0098  * gart table to be in system memory.
0099  */
0100 void radeon_gart_table_ram_free(struct radeon_device *rdev)
0101 {
0102     if (rdev->gart.ptr == NULL) {
0103         return;
0104     }
0105 #ifdef CONFIG_X86
0106     if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
0107         rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
0108         set_memory_wb((unsigned long)rdev->gart.ptr,
0109                   rdev->gart.table_size >> PAGE_SHIFT);
0110     }
0111 #endif
0112     dma_free_coherent(&rdev->pdev->dev, rdev->gart.table_size,
0113               (void *)rdev->gart.ptr, rdev->gart.table_addr);
0114     rdev->gart.ptr = NULL;
0115     rdev->gart.table_addr = 0;
0116 }
0117
0118 /**
0119  * radeon_gart_table_vram_alloc - allocate vram for gart page table
0120  *
0121  * @rdev: radeon_device pointer
0122  *
0123  * Allocate video memory for GART page table
0124  * (pcie r4xx, r5xx+).  These asics require the
0125  * gart table to be in video memory.
0126  * Returns 0 for success, error for failure.
0127  */
0128 int radeon_gart_table_vram_alloc(struct radeon_device *rdev)
0129 {
0130     int r;
0131
0132     if (rdev->gart.robj == NULL) {
0133         r = radeon_bo_create(rdev, rdev->gart.table_size,
0134                      PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
0135                      0, NULL, NULL, &rdev->gart.robj);
0136         if (r) {
0137             return r;
0138         }
0139     }
0140     return 0;
0141 }
0142
0143 /**
0144  * radeon_gart_table_vram_pin - pin gart page table in vram
0145  *
0146  * @rdev: radeon_device pointer
0147  *
0148  * Pin the GART page table in vram so it will not be moved
0149  * by the memory manager (pcie r4xx, r5xx+).  These asics require the
0150  * gart table to be in video memory.
0151  * Returns 0 for success, error for failure.
0152  */
0153 int radeon_gart_table_vram_pin(struct radeon_device *rdev)
0154 {
0155     uint64_t gpu_addr;
0156     int r;
0157
0158     r = radeon_bo_reserve(rdev->gart.robj, false);
0159     if (unlikely(r != 0))
0160         return r;
0161     r = radeon_bo_pin(rdev->gart.robj,
0162                 RADEON_GEM_DOMAIN_VRAM, &gpu_addr);
0163     if (r) {
0164         radeon_bo_unreserve(rdev->gart.robj);
0165         return r;
0166     }
0167     r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr);
0168     if (r)
0169         radeon_bo_unpin(rdev->gart.robj);
0170     radeon_bo_unreserve(rdev->gart.robj);
0171     rdev->gart.table_addr = gpu_addr;
0172
0173     if (!r) {
0174         int i;
0175
0176         /* We might have dropped some GART table updates while it wasn't
0177          * mapped, restore all entries
0178          */
0179         for (i = 0; i < rdev->gart.num_gpu_pages; i++)
0180             radeon_gart_set_page(rdev, i, rdev->gart.pages_entry[i]);
0181         mb();
0182         radeon_gart_tlb_flush(rdev);
0183     }
0184
0185     return r;
0186 }
0187
0188 /**
0189  * radeon_gart_table_vram_unpin - unpin gart page table in vram
0190  *
0191  * @rdev: radeon_device pointer
0192  *
0193  * Unpin the GART page table in vram (pcie r4xx, r5xx+).
0194  * These asics require the gart table to be in video memory.
0195  */
0196 void radeon_gart_table_vram_unpin(struct radeon_device *rdev)
0197 {
0198     int r;
0199
0200     if (rdev->gart.robj == NULL) {
0201         return;
0202     }
0203     r = radeon_bo_reserve(rdev->gart.robj, false);
0204     if (likely(r == 0)) {
0205         radeon_bo_kunmap(rdev->gart.robj);
0206         radeon_bo_unpin(rdev->gart.robj);
0207         radeon_bo_unreserve(rdev->gart.robj);
0208         rdev->gart.ptr = NULL;
0209     }
0210 }
0211
0212 /**
0213  * radeon_gart_table_vram_free - free gart page table vram
0214  *
0215  * @rdev: radeon_device pointer
0216  *
0217  * Free the video memory used for the GART page table
0218  * (pcie r4xx, r5xx+).  These asics require the gart table to
0219  * be in video memory.
0220  */
0221 void radeon_gart_table_vram_free(struct radeon_device *rdev)
0222 {
0223     if (rdev->gart.robj == NULL) {
0224         return;
0225     }
0226     radeon_bo_unref(&rdev->gart.robj);
0227 }
0228
0229 /*
0230  * Common gart functions.
0231  */
0232 /**
0233  * radeon_gart_unbind - unbind pages from the gart page table
0234  *
0235  * @rdev: radeon_device pointer
0236  * @offset: offset into the GPU's gart aperture
0237  * @pages: number of pages to unbind
0238  *
0239  * Unbinds the requested pages from the gart page table and
0240  * replaces them with the dummy page (all asics).
0241  */
0242 void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset,
0243             int pages)
0244 {
0245     unsigned t;
0246     unsigned p;
0247     int i, j;
0248
0249     if (!rdev->gart.ready) {
0250         WARN(1, "trying to unbind memory from uninitialized GART !\n");
0251         return;
0252     }
0253     t = offset / RADEON_GPU_PAGE_SIZE;
0254     p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
0255     for (i = 0; i < pages; i++, p++) {
0256         if (rdev->gart.pages[p]) {
0257             rdev->gart.pages[p] = NULL;
0258             for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
0259                 rdev->gart.pages_entry[t] = rdev->dummy_page.entry;
0260                 if (rdev->gart.ptr) {
0261                     radeon_gart_set_page(rdev, t,
0262                                  rdev->dummy_page.entry);
0263                 }
0264             }
0265         }
0266     }
0267     if (rdev->gart.ptr) {
0268         mb();
0269         radeon_gart_tlb_flush(rdev);
0270     }
0271 }
0272
0273 /**
0274  * radeon_gart_bind - bind pages into the gart page table
0275  *
0276  * @rdev: radeon_device pointer
0277  * @offset: offset into the GPU's gart aperture
0278  * @pages: number of pages to bind
0279  * @pagelist: pages to bind
0280  * @dma_addr: DMA addresses of pages
0281  * @flags: RADEON_GART_PAGE_* flags
0282  *
0283  * Binds the requested pages to the gart page table
0284  * (all asics).
0285  * Returns 0 for success, -EINVAL for failure.
0286  */
0287 int radeon_gart_bind(struct radeon_device *rdev, unsigned offset,
0288              int pages, struct page **pagelist, dma_addr_t *dma_addr,
0289              uint32_t flags)
0290 {
0291     unsigned t;
0292     unsigned p;
0293     uint64_t page_base, page_entry;
0294     int i, j;
0295
0296     if (!rdev->gart.ready) {
0297         WARN(1, "trying to bind memory to uninitialized GART !\n");
0298         return -EINVAL;
0299     }
0300     t = offset / RADEON_GPU_PAGE_SIZE;
0301     p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
0302
0303     for (i = 0; i < pages; i++, p++) {
0304         rdev->gart.pages[p] = pagelist ? pagelist[i] :
0305             rdev->dummy_page.page;
0306         page_base = dma_addr[i];
0307         for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
0308             page_entry = radeon_gart_get_page_entry(page_base, flags);
0309             rdev->gart.pages_entry[t] = page_entry;
0310             if (rdev->gart.ptr) {
0311                 radeon_gart_set_page(rdev, t, page_entry);
0312             }
0313             page_base += RADEON_GPU_PAGE_SIZE;
0314         }
0315     }
0316     if (rdev->gart.ptr) {
0317         mb();
0318         radeon_gart_tlb_flush(rdev);
0319     }
0320     return 0;
0321 }
0322
0323 /**
0324  * radeon_gart_init - init the driver info for managing the gart
0325  *
0326  * @rdev: radeon_device pointer
0327  *
0328  * Allocate the dummy page and init the gart driver info (all asics).
0329  * Returns 0 for success, error for failure.
0330  */
0331 int radeon_gart_init(struct radeon_device *rdev)
0332 {
0333     int r, i;
0334
0335     if (rdev->gart.pages) {
0336         return 0;
0337     }
0338     /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */
0339     if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) {
0340         DRM_ERROR("Page size is smaller than GPU page size!\n");
0341         return -EINVAL;
0342     }
0343     r = radeon_dummy_page_init(rdev);
0344     if (r)
0345         return r;
0346     /* Compute table size */
0347     rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE;
0348     rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE;
0349     DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
0350          rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages);
0351     /* Allocate pages table */
0352     rdev->gart.pages = vzalloc(array_size(sizeof(void *),
0353                    rdev->gart.num_cpu_pages));
0354     if (rdev->gart.pages == NULL) {
0355         radeon_gart_fini(rdev);
0356         return -ENOMEM;
0357     }
0358     rdev->gart.pages_entry = vmalloc(array_size(sizeof(uint64_t),
0359                             rdev->gart.num_gpu_pages));
0360     if (rdev->gart.pages_entry == NULL) {
0361         radeon_gart_fini(rdev);
0362         return -ENOMEM;
0363     }
0364     /* set GART entry to point to the dummy page by default */
0365     for (i = 0; i < rdev->gart.num_gpu_pages; i++)
0366         rdev->gart.pages_entry[i] = rdev->dummy_page.entry;
0367     return 0;
0368 }
0369
0370 /**
0371  * radeon_gart_fini - tear down the driver info for managing the gart
0372  *
0373  * @rdev: radeon_device pointer
0374  *
0375  * Tear down the gart driver info and free the dummy page (all asics).
0376  */
0377 void radeon_gart_fini(struct radeon_device *rdev)
0378 {
0379     if (rdev->gart.ready) {
0380         /* unbind pages */
0381         radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages);
0382     }
0383     rdev->gart.ready = false;
0384     vfree(rdev->gart.pages);
0385     vfree(rdev->gart.pages_entry);
0386     rdev->gart.pages = NULL;
0387     rdev->gart.pages_entry = NULL;
0388
0389     radeon_dummy_page_fini(rdev);
0390 }