drivers/fpga/dfl-afu-dma-region.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Driver for FPGA Accelerated Function Unit (AFU) DMA Region Management
0004  *
0005  * Copyright (C) 2017-2018 Intel Corporation, Inc.
0006  *
0007  * Authors:
0008  *   Wu Hao <hao.wu@intel.com>
0009  *   Xiao Guangrong <guangrong.xiao@linux.intel.com>
0010  */
0011
0012 #include <linux/dma-mapping.h>
0013 #include <linux/sched/signal.h>
0014 #include <linux/uaccess.h>
0015 #include <linux/mm.h>
0016
0017 #include "dfl-afu.h"
0018
0019 void afu_dma_region_init(struct dfl_feature_platform_data *pdata)
0020 {
0021     struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
0022
0023     afu->dma_regions = RB_ROOT;
0024 }
0025
0026 /**
0027  * afu_dma_pin_pages - pin pages of given dma memory region
0028  * @pdata: feature device platform data
0029  * @region: dma memory region to be pinned
0030  *
0031  * Pin all the pages of given dfl_afu_dma_region.
0032  * Return 0 for success or negative error code.
0033  */
0034 static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
0035                  struct dfl_afu_dma_region *region)
0036 {
0037     int npages = region->length >> PAGE_SHIFT;
0038     struct device *dev = &pdata->dev->dev;
0039     int ret, pinned;
0040
0041     ret = account_locked_vm(current->mm, npages, true);
0042     if (ret)
0043         return ret;
0044
0045     region->pages = kcalloc(npages, sizeof(struct page *), GFP_KERNEL);
0046     if (!region->pages) {
0047         ret = -ENOMEM;
0048         goto unlock_vm;
0049     }
0050
0051     pinned = pin_user_pages_fast(region->user_addr, npages, FOLL_WRITE,
0052                      region->pages);
0053     if (pinned < 0) {
0054         ret = pinned;
0055         goto free_pages;
0056     } else if (pinned != npages) {
0057         ret = -EFAULT;
0058         goto unpin_pages;
0059     }
0060
0061     dev_dbg(dev, "%d pages pinned\n", pinned);
0062
0063     return 0;
0064
0065 unpin_pages:
0066     unpin_user_pages(region->pages, pinned);
0067 free_pages:
0068     kfree(region->pages);
0069 unlock_vm:
0070     account_locked_vm(current->mm, npages, false);
0071     return ret;
0072 }
0073
0074 /**
0075  * afu_dma_unpin_pages - unpin pages of given dma memory region
0076  * @pdata: feature device platform data
0077  * @region: dma memory region to be unpinned
0078  *
0079  * Unpin all the pages of given dfl_afu_dma_region.
0080  * Return 0 for success or negative error code.
0081  */
0082 static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata,
0083                 struct dfl_afu_dma_region *region)
0084 {
0085     long npages = region->length >> PAGE_SHIFT;
0086     struct device *dev = &pdata->dev->dev;
0087
0088     unpin_user_pages(region->pages, npages);
0089     kfree(region->pages);
0090     account_locked_vm(current->mm, npages, false);
0091
0092     dev_dbg(dev, "%ld pages unpinned\n", npages);
0093 }
0094
0095 /**
0096  * afu_dma_check_continuous_pages - check if pages are continuous
0097  * @region: dma memory region
0098  *
0099  * Return true if pages of given dma memory region have continuous physical
0100  * address, otherwise return false.
0101  */
0102 static bool afu_dma_check_continuous_pages(struct dfl_afu_dma_region *region)
0103 {
0104     int npages = region->length >> PAGE_SHIFT;
0105     int i;
0106
0107     for (i = 0; i < npages - 1; i++)
0108         if (page_to_pfn(region->pages[i]) + 1 !=
0109                 page_to_pfn(region->pages[i + 1]))
0110             return false;
0111
0112     return true;
0113 }
0114
0115 /**
0116  * dma_region_check_iova - check if memory area is fully contained in the region
0117  * @region: dma memory region
0118  * @iova: address of the dma memory area
0119  * @size: size of the dma memory area
0120  *
0121  * Compare the dma memory area defined by @iova and @size with given dma region.
0122  * Return true if memory area is fully contained in the region, otherwise false.
0123  */
0124 static bool dma_region_check_iova(struct dfl_afu_dma_region *region,
0125                   u64 iova, u64 size)
0126 {
0127     if (!size && region->iova != iova)
0128         return false;
0129
0130     return (region->iova <= iova) &&
0131         (region->length + region->iova >= iova + size);
0132 }
0133
0134 /**
0135  * afu_dma_region_add - add given dma region to rbtree
0136  * @pdata: feature device platform data
0137  * @region: dma region to be added
0138  *
0139  * Return 0 for success, -EEXIST if dma region has already been added.
0140  *
0141  * Needs to be called with pdata->lock heold.
0142  */
0143 static int afu_dma_region_add(struct dfl_feature_platform_data *pdata,
0144                   struct dfl_afu_dma_region *region)
0145 {
0146     struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
0147     struct rb_node **new, *parent = NULL;
0148
0149     dev_dbg(&pdata->dev->dev, "add region (iova = %llx)\n",
0150         (unsigned long long)region->iova);
0151
0152     new = &afu->dma_regions.rb_node;
0153
0154     while (*new) {
0155         struct dfl_afu_dma_region *this;
0156
0157         this = container_of(*new, struct dfl_afu_dma_region, node);
0158
0159         parent = *new;
0160
0161         if (dma_region_check_iova(this, region->iova, region->length))
0162             return -EEXIST;
0163
0164         if (region->iova < this->iova)
0165             new = &((*new)->rb_left);
0166         else if (region->iova > this->iova)
0167             new = &((*new)->rb_right);
0168         else
0169             return -EEXIST;
0170     }
0171
0172     rb_link_node(&region->node, parent, new);
0173     rb_insert_color(&region->node, &afu->dma_regions);
0174
0175     return 0;
0176 }
0177
0178 /**
0179  * afu_dma_region_remove - remove given dma region from rbtree
0180  * @pdata: feature device platform data
0181  * @region: dma region to be removed
0182  *
0183  * Needs to be called with pdata->lock heold.
0184  */
0185 static void afu_dma_region_remove(struct dfl_feature_platform_data *pdata,
0186                   struct dfl_afu_dma_region *region)
0187 {
0188     struct dfl_afu *afu;
0189
0190     dev_dbg(&pdata->dev->dev, "del region (iova = %llx)\n",
0191         (unsigned long long)region->iova);
0192
0193     afu = dfl_fpga_pdata_get_private(pdata);
0194     rb_erase(&region->node, &afu->dma_regions);
0195 }
0196
0197 /**
0198  * afu_dma_region_destroy - destroy all regions in rbtree
0199  * @pdata: feature device platform data
0200  *
0201  * Needs to be called with pdata->lock heold.
0202  */
0203 void afu_dma_region_destroy(struct dfl_feature_platform_data *pdata)
0204 {
0205     struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
0206     struct rb_node *node = rb_first(&afu->dma_regions);
0207     struct dfl_afu_dma_region *region;
0208
0209     while (node) {
0210         region = container_of(node, struct dfl_afu_dma_region, node);
0211
0212         dev_dbg(&pdata->dev->dev, "del region (iova = %llx)\n",
0213             (unsigned long long)region->iova);
0214
0215         rb_erase(node, &afu->dma_regions);
0216
0217         if (region->iova)
0218             dma_unmap_page(dfl_fpga_pdata_to_parent(pdata),
0219                        region->iova, region->length,
0220                        DMA_BIDIRECTIONAL);
0221
0222         if (region->pages)
0223             afu_dma_unpin_pages(pdata, region);
0224
0225         node = rb_next(node);
0226         kfree(region);
0227     }
0228 }
0229
0230 /**
0231  * afu_dma_region_find - find the dma region from rbtree based on iova and size
0232  * @pdata: feature device platform data
0233  * @iova: address of the dma memory area
0234  * @size: size of the dma memory area
0235  *
0236  * It finds the dma region from the rbtree based on @iova and @size:
0237  * - if @size == 0, it finds the dma region which starts from @iova
0238  * - otherwise, it finds the dma region which fully contains
0239  *   [@iova, @iova+size)
0240  * If nothing is matched returns NULL.
0241  *
0242  * Needs to be called with pdata->lock held.
0243  */
0244 struct dfl_afu_dma_region *
0245 afu_dma_region_find(struct dfl_feature_platform_data *pdata, u64 iova, u64 size)
0246 {
0247     struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
0248     struct rb_node *node = afu->dma_regions.rb_node;
0249     struct device *dev = &pdata->dev->dev;
0250
0251     while (node) {
0252         struct dfl_afu_dma_region *region;
0253
0254         region = container_of(node, struct dfl_afu_dma_region, node);
0255
0256         if (dma_region_check_iova(region, iova, size)) {
0257             dev_dbg(dev, "find region (iova = %llx)\n",
0258                 (unsigned long long)region->iova);
0259             return region;
0260         }
0261
0262         if (iova < region->iova)
0263             node = node->rb_left;
0264         else if (iova > region->iova)
0265             node = node->rb_right;
0266         else
0267             /* the iova region is not fully covered. */
0268             break;
0269     }
0270
0271     dev_dbg(dev, "region with iova %llx and size %llx is not found\n",
0272         (unsigned long long)iova, (unsigned long long)size);
0273
0274     return NULL;
0275 }
0276
0277 /**
0278  * afu_dma_region_find_iova - find the dma region from rbtree by iova
0279  * @pdata: feature device platform data
0280  * @iova: address of the dma region
0281  *
0282  * Needs to be called with pdata->lock held.
0283  */
0284 static struct dfl_afu_dma_region *
0285 afu_dma_region_find_iova(struct dfl_feature_platform_data *pdata, u64 iova)
0286 {
0287     return afu_dma_region_find(pdata, iova, 0);
0288 }
0289
0290 /**
0291  * afu_dma_map_region - map memory region for dma
0292  * @pdata: feature device platform data
0293  * @user_addr: address of the memory region
0294  * @length: size of the memory region
0295  * @iova: pointer of iova address
0296  *
0297  * Map memory region defined by @user_addr and @length, and return dma address
0298  * of the memory region via @iova.
0299  * Return 0 for success, otherwise error code.
0300  */
0301 int afu_dma_map_region(struct dfl_feature_platform_data *pdata,
0302                u64 user_addr, u64 length, u64 *iova)
0303 {
0304     struct dfl_afu_dma_region *region;
0305     int ret;
0306
0307     /*
0308      * Check Inputs, only accept page-aligned user memory region with
0309      * valid length.
0310      */
0311     if (!PAGE_ALIGNED(user_addr) || !PAGE_ALIGNED(length) || !length)
0312         return -EINVAL;
0313
0314     /* Check overflow */
0315     if (user_addr + length < user_addr)
0316         return -EINVAL;
0317
0318     region = kzalloc(sizeof(*region), GFP_KERNEL);
0319     if (!region)
0320         return -ENOMEM;
0321
0322     region->user_addr = user_addr;
0323     region->length = length;
0324
0325     /* Pin the user memory region */
0326     ret = afu_dma_pin_pages(pdata, region);
0327     if (ret) {
0328         dev_err(&pdata->dev->dev, "failed to pin memory region\n");
0329         goto free_region;
0330     }
0331
0332     /* Only accept continuous pages, return error else */
0333     if (!afu_dma_check_continuous_pages(region)) {
0334         dev_err(&pdata->dev->dev, "pages are not continuous\n");
0335         ret = -EINVAL;
0336         goto unpin_pages;
0337     }
0338
0339     /* As pages are continuous then start to do DMA mapping */
0340     region->iova = dma_map_page(dfl_fpga_pdata_to_parent(pdata),
0341                     region->pages[0], 0,
0342                     region->length,
0343                     DMA_BIDIRECTIONAL);
0344     if (dma_mapping_error(dfl_fpga_pdata_to_parent(pdata), region->iova)) {
0345         dev_err(&pdata->dev->dev, "failed to map for dma\n");
0346         ret = -EFAULT;
0347         goto unpin_pages;
0348     }
0349
0350     *iova = region->iova;
0351
0352     mutex_lock(&pdata->lock);
0353     ret = afu_dma_region_add(pdata, region);
0354     mutex_unlock(&pdata->lock);
0355     if (ret) {
0356         dev_err(&pdata->dev->dev, "failed to add dma region\n");
0357         goto unmap_dma;
0358     }
0359
0360     return 0;
0361
0362 unmap_dma:
0363     dma_unmap_page(dfl_fpga_pdata_to_parent(pdata),
0364                region->iova, region->length, DMA_BIDIRECTIONAL);
0365 unpin_pages:
0366     afu_dma_unpin_pages(pdata, region);
0367 free_region:
0368     kfree(region);
0369     return ret;
0370 }
0371
0372 /**
0373  * afu_dma_unmap_region - unmap dma memory region
0374  * @pdata: feature device platform data
0375  * @iova: dma address of the region
0376  *
0377  * Unmap dma memory region based on @iova.
0378  * Return 0 for success, otherwise error code.
0379  */
0380 int afu_dma_unmap_region(struct dfl_feature_platform_data *pdata, u64 iova)
0381 {
0382     struct dfl_afu_dma_region *region;
0383
0384     mutex_lock(&pdata->lock);
0385     region = afu_dma_region_find_iova(pdata, iova);
0386     if (!region) {
0387         mutex_unlock(&pdata->lock);
0388         return -EINVAL;
0389     }
0390
0391     if (region->in_use) {
0392         mutex_unlock(&pdata->lock);
0393         return -EBUSY;
0394     }
0395
0396     afu_dma_region_remove(pdata, region);
0397     mutex_unlock(&pdata->lock);
0398
0399     dma_unmap_page(dfl_fpga_pdata_to_parent(pdata),
0400                region->iova, region->length, DMA_BIDIRECTIONAL);
0401     afu_dma_unpin_pages(pdata, region);
0402     kfree(region);
0403
0404     return 0;
0405 }