0001
0002 #include <linux/cred.h>
0003 #include <linux/device.h>
0004 #include <linux/dma-buf.h>
0005 #include <linux/highmem.h>
0006 #include <linux/init.h>
0007 #include <linux/kernel.h>
0008 #include <linux/memfd.h>
0009 #include <linux/miscdevice.h>
0010 #include <linux/module.h>
0011 #include <linux/shmem_fs.h>
0012 #include <linux/slab.h>
0013 #include <linux/udmabuf.h>
0014 #include <linux/hugetlb.h>
0015
0016 static int list_limit = 1024;
0017 module_param(list_limit, int, 0644);
0018 MODULE_PARM_DESC(list_limit, "udmabuf_create_list->count limit. Default is 1024.");
0019
0020 static int size_limit_mb = 64;
0021 module_param(size_limit_mb, int, 0644);
0022 MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64.");
0023
0024 struct udmabuf {
0025 pgoff_t pagecount;
0026 struct page **pages;
0027 struct sg_table *sg;
0028 struct miscdevice *device;
0029 };
0030
0031 static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)
0032 {
0033 struct vm_area_struct *vma = vmf->vma;
0034 struct udmabuf *ubuf = vma->vm_private_data;
0035 pgoff_t pgoff = vmf->pgoff;
0036
0037 if (pgoff >= ubuf->pagecount)
0038 return VM_FAULT_SIGBUS;
0039 vmf->page = ubuf->pages[pgoff];
0040 get_page(vmf->page);
0041 return 0;
0042 }
0043
0044 static const struct vm_operations_struct udmabuf_vm_ops = {
0045 .fault = udmabuf_vm_fault,
0046 };
0047
0048 static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma)
0049 {
0050 struct udmabuf *ubuf = buf->priv;
0051
0052 if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
0053 return -EINVAL;
0054
0055 vma->vm_ops = &udmabuf_vm_ops;
0056 vma->vm_private_data = ubuf;
0057 return 0;
0058 }
0059
0060 static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf,
0061 enum dma_data_direction direction)
0062 {
0063 struct udmabuf *ubuf = buf->priv;
0064 struct sg_table *sg;
0065 int ret;
0066
0067 sg = kzalloc(sizeof(*sg), GFP_KERNEL);
0068 if (!sg)
0069 return ERR_PTR(-ENOMEM);
0070 ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->pagecount,
0071 0, ubuf->pagecount << PAGE_SHIFT,
0072 GFP_KERNEL);
0073 if (ret < 0)
0074 goto err;
0075 ret = dma_map_sgtable(dev, sg, direction, 0);
0076 if (ret < 0)
0077 goto err;
0078 return sg;
0079
0080 err:
0081 sg_free_table(sg);
0082 kfree(sg);
0083 return ERR_PTR(ret);
0084 }
0085
0086 static void put_sg_table(struct device *dev, struct sg_table *sg,
0087 enum dma_data_direction direction)
0088 {
0089 dma_unmap_sgtable(dev, sg, direction, 0);
0090 sg_free_table(sg);
0091 kfree(sg);
0092 }
0093
0094 static struct sg_table *map_udmabuf(struct dma_buf_attachment *at,
0095 enum dma_data_direction direction)
0096 {
0097 return get_sg_table(at->dev, at->dmabuf, direction);
0098 }
0099
0100 static void unmap_udmabuf(struct dma_buf_attachment *at,
0101 struct sg_table *sg,
0102 enum dma_data_direction direction)
0103 {
0104 return put_sg_table(at->dev, sg, direction);
0105 }
0106
0107 static void release_udmabuf(struct dma_buf *buf)
0108 {
0109 struct udmabuf *ubuf = buf->priv;
0110 struct device *dev = ubuf->device->this_device;
0111 pgoff_t pg;
0112
0113 if (ubuf->sg)
0114 put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL);
0115
0116 for (pg = 0; pg < ubuf->pagecount; pg++)
0117 put_page(ubuf->pages[pg]);
0118 kfree(ubuf->pages);
0119 kfree(ubuf);
0120 }
0121
0122 static int begin_cpu_udmabuf(struct dma_buf *buf,
0123 enum dma_data_direction direction)
0124 {
0125 struct udmabuf *ubuf = buf->priv;
0126 struct device *dev = ubuf->device->this_device;
0127
0128 if (!ubuf->sg) {
0129 ubuf->sg = get_sg_table(dev, buf, direction);
0130 if (IS_ERR(ubuf->sg))
0131 return PTR_ERR(ubuf->sg);
0132 } else {
0133 dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents,
0134 direction);
0135 }
0136
0137 return 0;
0138 }
0139
0140 static int end_cpu_udmabuf(struct dma_buf *buf,
0141 enum dma_data_direction direction)
0142 {
0143 struct udmabuf *ubuf = buf->priv;
0144 struct device *dev = ubuf->device->this_device;
0145
0146 if (!ubuf->sg)
0147 return -EINVAL;
0148
0149 dma_sync_sg_for_device(dev, ubuf->sg->sgl, ubuf->sg->nents, direction);
0150 return 0;
0151 }
0152
0153 static const struct dma_buf_ops udmabuf_ops = {
0154 .cache_sgt_mapping = true,
0155 .map_dma_buf = map_udmabuf,
0156 .unmap_dma_buf = unmap_udmabuf,
0157 .release = release_udmabuf,
0158 .mmap = mmap_udmabuf,
0159 .begin_cpu_access = begin_cpu_udmabuf,
0160 .end_cpu_access = end_cpu_udmabuf,
0161 };
0162
0163 #define SEALS_WANTED (F_SEAL_SHRINK)
0164 #define SEALS_DENIED (F_SEAL_WRITE)
0165
0166 static long udmabuf_create(struct miscdevice *device,
0167 struct udmabuf_create_list *head,
0168 struct udmabuf_create_item *list)
0169 {
0170 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
0171 struct file *memfd = NULL;
0172 struct address_space *mapping = NULL;
0173 struct udmabuf *ubuf;
0174 struct dma_buf *buf;
0175 pgoff_t pgoff, pgcnt, pgidx, pgbuf = 0, pglimit;
0176 struct page *page, *hpage = NULL;
0177 pgoff_t subpgoff, maxsubpgs;
0178 struct hstate *hpstate;
0179 int seals, ret = -EINVAL;
0180 u32 i, flags;
0181
0182 ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL);
0183 if (!ubuf)
0184 return -ENOMEM;
0185
0186 pglimit = (size_limit_mb * 1024 * 1024) >> PAGE_SHIFT;
0187 for (i = 0; i < head->count; i++) {
0188 if (!IS_ALIGNED(list[i].offset, PAGE_SIZE))
0189 goto err;
0190 if (!IS_ALIGNED(list[i].size, PAGE_SIZE))
0191 goto err;
0192 ubuf->pagecount += list[i].size >> PAGE_SHIFT;
0193 if (ubuf->pagecount > pglimit)
0194 goto err;
0195 }
0196
0197 if (!ubuf->pagecount)
0198 goto err;
0199
0200 ubuf->pages = kmalloc_array(ubuf->pagecount, sizeof(*ubuf->pages),
0201 GFP_KERNEL);
0202 if (!ubuf->pages) {
0203 ret = -ENOMEM;
0204 goto err;
0205 }
0206
0207 pgbuf = 0;
0208 for (i = 0; i < head->count; i++) {
0209 ret = -EBADFD;
0210 memfd = fget(list[i].memfd);
0211 if (!memfd)
0212 goto err;
0213 mapping = file_inode(memfd)->i_mapping;
0214 if (!shmem_mapping(mapping) && !is_file_hugepages(memfd))
0215 goto err;
0216 seals = memfd_fcntl(memfd, F_GET_SEALS, 0);
0217 if (seals == -EINVAL)
0218 goto err;
0219 ret = -EINVAL;
0220 if ((seals & SEALS_WANTED) != SEALS_WANTED ||
0221 (seals & SEALS_DENIED) != 0)
0222 goto err;
0223 pgoff = list[i].offset >> PAGE_SHIFT;
0224 pgcnt = list[i].size >> PAGE_SHIFT;
0225 if (is_file_hugepages(memfd)) {
0226 hpstate = hstate_file(memfd);
0227 pgoff = list[i].offset >> huge_page_shift(hpstate);
0228 subpgoff = (list[i].offset &
0229 ~huge_page_mask(hpstate)) >> PAGE_SHIFT;
0230 maxsubpgs = huge_page_size(hpstate) >> PAGE_SHIFT;
0231 }
0232 for (pgidx = 0; pgidx < pgcnt; pgidx++) {
0233 if (is_file_hugepages(memfd)) {
0234 if (!hpage) {
0235 hpage = find_get_page_flags(mapping, pgoff,
0236 FGP_ACCESSED);
0237 if (!hpage) {
0238 ret = -EINVAL;
0239 goto err;
0240 }
0241 }
0242 page = hpage + subpgoff;
0243 get_page(page);
0244 subpgoff++;
0245 if (subpgoff == maxsubpgs) {
0246 put_page(hpage);
0247 hpage = NULL;
0248 subpgoff = 0;
0249 pgoff++;
0250 }
0251 } else {
0252 page = shmem_read_mapping_page(mapping,
0253 pgoff + pgidx);
0254 if (IS_ERR(page)) {
0255 ret = PTR_ERR(page);
0256 goto err;
0257 }
0258 }
0259 ubuf->pages[pgbuf++] = page;
0260 }
0261 fput(memfd);
0262 memfd = NULL;
0263 if (hpage) {
0264 put_page(hpage);
0265 hpage = NULL;
0266 }
0267 }
0268
0269 exp_info.ops = &udmabuf_ops;
0270 exp_info.size = ubuf->pagecount << PAGE_SHIFT;
0271 exp_info.priv = ubuf;
0272 exp_info.flags = O_RDWR;
0273
0274 ubuf->device = device;
0275 buf = dma_buf_export(&exp_info);
0276 if (IS_ERR(buf)) {
0277 ret = PTR_ERR(buf);
0278 goto err;
0279 }
0280
0281 flags = 0;
0282 if (head->flags & UDMABUF_FLAGS_CLOEXEC)
0283 flags |= O_CLOEXEC;
0284 return dma_buf_fd(buf, flags);
0285
0286 err:
0287 while (pgbuf > 0)
0288 put_page(ubuf->pages[--pgbuf]);
0289 if (memfd)
0290 fput(memfd);
0291 kfree(ubuf->pages);
0292 kfree(ubuf);
0293 return ret;
0294 }
0295
0296 static long udmabuf_ioctl_create(struct file *filp, unsigned long arg)
0297 {
0298 struct udmabuf_create create;
0299 struct udmabuf_create_list head;
0300 struct udmabuf_create_item list;
0301
0302 if (copy_from_user(&create, (void __user *)arg,
0303 sizeof(create)))
0304 return -EFAULT;
0305
0306 head.flags = create.flags;
0307 head.count = 1;
0308 list.memfd = create.memfd;
0309 list.offset = create.offset;
0310 list.size = create.size;
0311
0312 return udmabuf_create(filp->private_data, &head, &list);
0313 }
0314
0315 static long udmabuf_ioctl_create_list(struct file *filp, unsigned long arg)
0316 {
0317 struct udmabuf_create_list head;
0318 struct udmabuf_create_item *list;
0319 int ret = -EINVAL;
0320 u32 lsize;
0321
0322 if (copy_from_user(&head, (void __user *)arg, sizeof(head)))
0323 return -EFAULT;
0324 if (head.count > list_limit)
0325 return -EINVAL;
0326 lsize = sizeof(struct udmabuf_create_item) * head.count;
0327 list = memdup_user((void __user *)(arg + sizeof(head)), lsize);
0328 if (IS_ERR(list))
0329 return PTR_ERR(list);
0330
0331 ret = udmabuf_create(filp->private_data, &head, list);
0332 kfree(list);
0333 return ret;
0334 }
0335
0336 static long udmabuf_ioctl(struct file *filp, unsigned int ioctl,
0337 unsigned long arg)
0338 {
0339 long ret;
0340
0341 switch (ioctl) {
0342 case UDMABUF_CREATE:
0343 ret = udmabuf_ioctl_create(filp, arg);
0344 break;
0345 case UDMABUF_CREATE_LIST:
0346 ret = udmabuf_ioctl_create_list(filp, arg);
0347 break;
0348 default:
0349 ret = -ENOTTY;
0350 break;
0351 }
0352 return ret;
0353 }
0354
0355 static const struct file_operations udmabuf_fops = {
0356 .owner = THIS_MODULE,
0357 .unlocked_ioctl = udmabuf_ioctl,
0358 #ifdef CONFIG_COMPAT
0359 .compat_ioctl = udmabuf_ioctl,
0360 #endif
0361 };
0362
0363 static struct miscdevice udmabuf_misc = {
0364 .minor = MISC_DYNAMIC_MINOR,
0365 .name = "udmabuf",
0366 .fops = &udmabuf_fops,
0367 };
0368
0369 static int __init udmabuf_dev_init(void)
0370 {
0371 int ret;
0372
0373 ret = misc_register(&udmabuf_misc);
0374 if (ret < 0) {
0375 pr_err("Could not initialize udmabuf device\n");
0376 return ret;
0377 }
0378
0379 ret = dma_coerce_mask_and_coherent(udmabuf_misc.this_device,
0380 DMA_BIT_MASK(64));
0381 if (ret < 0) {
0382 pr_err("Could not setup DMA mask for udmabuf device\n");
0383 misc_deregister(&udmabuf_misc);
0384 return ret;
0385 }
0386
0387 return 0;
0388 }
0389
0390 static void __exit udmabuf_dev_exit(void)
0391 {
0392 misc_deregister(&udmabuf_misc);
0393 }
0394
0395 module_init(udmabuf_dev_init)
0396 module_exit(udmabuf_dev_exit)
0397
0398 MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>");
0399 MODULE_LICENSE("GPL v2");
