virt/kvm/vfio.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * VFIO-KVM bridge pseudo device
0004  *
0005  * Copyright (C) 2013 Red Hat, Inc.  All rights reserved.
0006  *     Author: Alex Williamson <alex.williamson@redhat.com>
0007  */
0008
0009 #include <linux/errno.h>
0010 #include <linux/file.h>
0011 #include <linux/kvm_host.h>
0012 #include <linux/list.h>
0013 #include <linux/module.h>
0014 #include <linux/mutex.h>
0015 #include <linux/slab.h>
0016 #include <linux/uaccess.h>
0017 #include <linux/vfio.h>
0018 #include "vfio.h"
0019
0020 #ifdef CONFIG_SPAPR_TCE_IOMMU
0021 #include <asm/kvm_ppc.h>
0022 #endif
0023
0024 struct kvm_vfio_group {
0025     struct list_head node;
0026     struct file *file;
0027 };
0028
0029 struct kvm_vfio {
0030     struct list_head group_list;
0031     struct mutex lock;
0032     bool noncoherent;
0033 };
0034
0035 static void kvm_vfio_file_set_kvm(struct file *file, struct kvm *kvm)
0036 {
0037     void (*fn)(struct file *file, struct kvm *kvm);
0038
0039     fn = symbol_get(vfio_file_set_kvm);
0040     if (!fn)
0041         return;
0042
0043     fn(file, kvm);
0044
0045     symbol_put(vfio_file_set_kvm);
0046 }
0047
0048 static bool kvm_vfio_file_enforced_coherent(struct file *file)
0049 {
0050     bool (*fn)(struct file *file);
0051     bool ret;
0052
0053     fn = symbol_get(vfio_file_enforced_coherent);
0054     if (!fn)
0055         return false;
0056
0057     ret = fn(file);
0058
0059     symbol_put(vfio_file_enforced_coherent);
0060
0061     return ret;
0062 }
0063
0064 static struct iommu_group *kvm_vfio_file_iommu_group(struct file *file)
0065 {
0066     struct iommu_group *(*fn)(struct file *file);
0067     struct iommu_group *ret;
0068
0069     fn = symbol_get(vfio_file_iommu_group);
0070     if (!fn)
0071         return NULL;
0072
0073     ret = fn(file);
0074
0075     symbol_put(vfio_file_iommu_group);
0076
0077     return ret;
0078 }
0079
0080 #ifdef CONFIG_SPAPR_TCE_IOMMU
0081 static void kvm_spapr_tce_release_vfio_group(struct kvm *kvm,
0082                          struct kvm_vfio_group *kvg)
0083 {
0084     struct iommu_group *grp = kvm_vfio_file_iommu_group(kvg->file);
0085
0086     if (WARN_ON_ONCE(!grp))
0087         return;
0088
0089     kvm_spapr_tce_release_iommu_group(kvm, grp);
0090 }
0091 #endif
0092
0093 /*
0094  * Groups can use the same or different IOMMU domains.  If the same then
0095  * adding a new group may change the coherency of groups we've previously
0096  * been told about.  We don't want to care about any of that so we retest
0097  * each group and bail as soon as we find one that's noncoherent.  This
0098  * means we only ever [un]register_noncoherent_dma once for the whole device.
0099  */
0100 static void kvm_vfio_update_coherency(struct kvm_device *dev)
0101 {
0102     struct kvm_vfio *kv = dev->private;
0103     bool noncoherent = false;
0104     struct kvm_vfio_group *kvg;
0105
0106     mutex_lock(&kv->lock);
0107
0108     list_for_each_entry(kvg, &kv->group_list, node) {
0109         if (!kvm_vfio_file_enforced_coherent(kvg->file)) {
0110             noncoherent = true;
0111             break;
0112         }
0113     }
0114
0115     if (noncoherent != kv->noncoherent) {
0116         kv->noncoherent = noncoherent;
0117
0118         if (kv->noncoherent)
0119             kvm_arch_register_noncoherent_dma(dev->kvm);
0120         else
0121             kvm_arch_unregister_noncoherent_dma(dev->kvm);
0122     }
0123
0124     mutex_unlock(&kv->lock);
0125 }
0126
0127 static int kvm_vfio_group_add(struct kvm_device *dev, unsigned int fd)
0128 {
0129     struct kvm_vfio *kv = dev->private;
0130     struct kvm_vfio_group *kvg;
0131     struct file *filp;
0132     int ret;
0133
0134     filp = fget(fd);
0135     if (!filp)
0136         return -EBADF;
0137
0138     /* Ensure the FD is a vfio group FD.*/
0139     if (!kvm_vfio_file_iommu_group(filp)) {
0140         ret = -EINVAL;
0141         goto err_fput;
0142     }
0143
0144     mutex_lock(&kv->lock);
0145
0146     list_for_each_entry(kvg, &kv->group_list, node) {
0147         if (kvg->file == filp) {
0148             ret = -EEXIST;
0149             goto err_unlock;
0150         }
0151     }
0152
0153     kvg = kzalloc(sizeof(*kvg), GFP_KERNEL_ACCOUNT);
0154     if (!kvg) {
0155         ret = -ENOMEM;
0156         goto err_unlock;
0157     }
0158
0159     kvg->file = filp;
0160     list_add_tail(&kvg->node, &kv->group_list);
0161
0162     kvm_arch_start_assignment(dev->kvm);
0163
0164     mutex_unlock(&kv->lock);
0165
0166     kvm_vfio_file_set_kvm(kvg->file, dev->kvm);
0167     kvm_vfio_update_coherency(dev);
0168
0169     return 0;
0170 err_unlock:
0171     mutex_unlock(&kv->lock);
0172 err_fput:
0173     fput(filp);
0174     return ret;
0175 }
0176
0177 static int kvm_vfio_group_del(struct kvm_device *dev, unsigned int fd)
0178 {
0179     struct kvm_vfio *kv = dev->private;
0180     struct kvm_vfio_group *kvg;
0181     struct fd f;
0182     int ret;
0183
0184     f = fdget(fd);
0185     if (!f.file)
0186         return -EBADF;
0187
0188     ret = -ENOENT;
0189
0190     mutex_lock(&kv->lock);
0191
0192     list_for_each_entry(kvg, &kv->group_list, node) {
0193         if (kvg->file != f.file)
0194             continue;
0195
0196         list_del(&kvg->node);
0197         kvm_arch_end_assignment(dev->kvm);
0198 #ifdef CONFIG_SPAPR_TCE_IOMMU
0199         kvm_spapr_tce_release_vfio_group(dev->kvm, kvg);
0200 #endif
0201         kvm_vfio_file_set_kvm(kvg->file, NULL);
0202         fput(kvg->file);
0203         kfree(kvg);
0204         ret = 0;
0205         break;
0206     }
0207
0208     mutex_unlock(&kv->lock);
0209
0210     fdput(f);
0211
0212     kvm_vfio_update_coherency(dev);
0213
0214     return ret;
0215 }
0216
0217 #ifdef CONFIG_SPAPR_TCE_IOMMU
0218 static int kvm_vfio_group_set_spapr_tce(struct kvm_device *dev,
0219                     void __user *arg)
0220 {
0221     struct kvm_vfio_spapr_tce param;
0222     struct kvm_vfio *kv = dev->private;
0223     struct kvm_vfio_group *kvg;
0224     struct fd f;
0225     int ret;
0226
0227     if (copy_from_user(&param, arg, sizeof(struct kvm_vfio_spapr_tce)))
0228         return -EFAULT;
0229
0230     f = fdget(param.groupfd);
0231     if (!f.file)
0232         return -EBADF;
0233
0234     ret = -ENOENT;
0235
0236     mutex_lock(&kv->lock);
0237
0238     list_for_each_entry(kvg, &kv->group_list, node) {
0239         struct iommu_group *grp;
0240
0241         if (kvg->file != f.file)
0242             continue;
0243
0244         grp = kvm_vfio_file_iommu_group(kvg->file);
0245         if (WARN_ON_ONCE(!grp)) {
0246             ret = -EIO;
0247             goto err_fdput;
0248         }
0249
0250         ret = kvm_spapr_tce_attach_iommu_group(dev->kvm, param.tablefd,
0251                                grp);
0252         break;
0253     }
0254
0255 err_fdput:
0256     mutex_unlock(&kv->lock);
0257     fdput(f);
0258     return ret;
0259 }
0260 #endif
0261
0262 static int kvm_vfio_set_group(struct kvm_device *dev, long attr,
0263                   void __user *arg)
0264 {
0265     int32_t __user *argp = arg;
0266     int32_t fd;
0267
0268     switch (attr) {
0269     case KVM_DEV_VFIO_GROUP_ADD:
0270         if (get_user(fd, argp))
0271             return -EFAULT;
0272         return kvm_vfio_group_add(dev, fd);
0273
0274     case KVM_DEV_VFIO_GROUP_DEL:
0275         if (get_user(fd, argp))
0276             return -EFAULT;
0277         return kvm_vfio_group_del(dev, fd);
0278
0279 #ifdef CONFIG_SPAPR_TCE_IOMMU
0280     case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE:
0281         return kvm_vfio_group_set_spapr_tce(dev, arg);
0282 #endif
0283     }
0284
0285     return -ENXIO;
0286 }
0287
0288 static int kvm_vfio_set_attr(struct kvm_device *dev,
0289                  struct kvm_device_attr *attr)
0290 {
0291     switch (attr->group) {
0292     case KVM_DEV_VFIO_GROUP:
0293         return kvm_vfio_set_group(dev, attr->attr,
0294                       u64_to_user_ptr(attr->addr));
0295     }
0296
0297     return -ENXIO;
0298 }
0299
0300 static int kvm_vfio_has_attr(struct kvm_device *dev,
0301                  struct kvm_device_attr *attr)
0302 {
0303     switch (attr->group) {
0304     case KVM_DEV_VFIO_GROUP:
0305         switch (attr->attr) {
0306         case KVM_DEV_VFIO_GROUP_ADD:
0307         case KVM_DEV_VFIO_GROUP_DEL:
0308 #ifdef CONFIG_SPAPR_TCE_IOMMU
0309         case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE:
0310 #endif
0311             return 0;
0312         }
0313
0314         break;
0315     }
0316
0317     return -ENXIO;
0318 }
0319
0320 static void kvm_vfio_destroy(struct kvm_device *dev)
0321 {
0322     struct kvm_vfio *kv = dev->private;
0323     struct kvm_vfio_group *kvg, *tmp;
0324
0325     list_for_each_entry_safe(kvg, tmp, &kv->group_list, node) {
0326 #ifdef CONFIG_SPAPR_TCE_IOMMU
0327         kvm_spapr_tce_release_vfio_group(dev->kvm, kvg);
0328 #endif
0329         kvm_vfio_file_set_kvm(kvg->file, NULL);
0330         fput(kvg->file);
0331         list_del(&kvg->node);
0332         kfree(kvg);
0333         kvm_arch_end_assignment(dev->kvm);
0334     }
0335
0336     kvm_vfio_update_coherency(dev);
0337
0338     kfree(kv);
0339     kfree(dev); /* alloc by kvm_ioctl_create_device, free by .destroy */
0340 }
0341
0342 static int kvm_vfio_create(struct kvm_device *dev, u32 type);
0343
0344 static struct kvm_device_ops kvm_vfio_ops = {
0345     .name = "kvm-vfio",
0346     .create = kvm_vfio_create,
0347     .destroy = kvm_vfio_destroy,
0348     .set_attr = kvm_vfio_set_attr,
0349     .has_attr = kvm_vfio_has_attr,
0350 };
0351
0352 static int kvm_vfio_create(struct kvm_device *dev, u32 type)
0353 {
0354     struct kvm_device *tmp;
0355     struct kvm_vfio *kv;
0356
0357     /* Only one VFIO "device" per VM */
0358     list_for_each_entry(tmp, &dev->kvm->devices, vm_node)
0359         if (tmp->ops == &kvm_vfio_ops)
0360             return -EBUSY;
0361
0362     kv = kzalloc(sizeof(*kv), GFP_KERNEL_ACCOUNT);
0363     if (!kv)
0364         return -ENOMEM;
0365
0366     INIT_LIST_HEAD(&kv->group_list);
0367     mutex_init(&kv->lock);
0368
0369     dev->private = kv;
0370
0371     return 0;
0372 }
0373
0374 int kvm_vfio_ops_init(void)
0375 {
0376     return kvm_register_device_ops(&kvm_vfio_ops, KVM_DEV_TYPE_VFIO);
0377 }
0378
0379 void kvm_vfio_ops_exit(void)
0380 {
0381     kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO);
0382 }