misc/ocxl/file.c

0001 // SPDX-License-Identifier: GPL-2.0+
0002 // Copyright 2017 IBM Corp.
0003 #include <linux/fs.h>
0004 #include <linux/poll.h>
0005 #include <linux/sched/signal.h>
0006 #include <linux/eventfd.h>
0007 #include <linux/uaccess.h>
0008 #include <uapi/misc/ocxl.h>
0009 #include <asm/reg.h>
0010 #include <asm/switch_to.h>
0011 #include "ocxl_internal.h"
0012
0013
0014 #define OCXL_NUM_MINORS 256 /* Total to reserve */
0015
0016 static dev_t ocxl_dev;
0017 static struct class *ocxl_class;
0018 static DEFINE_MUTEX(minors_idr_lock);
0019 static struct idr minors_idr;
0020
0021 static struct ocxl_file_info *find_and_get_file_info(dev_t devno)
0022 {
0023     struct ocxl_file_info *info;
0024
0025     mutex_lock(&minors_idr_lock);
0026     info = idr_find(&minors_idr, MINOR(devno));
0027     if (info)
0028         get_device(&info->dev);
0029     mutex_unlock(&minors_idr_lock);
0030     return info;
0031 }
0032
0033 static int allocate_minor(struct ocxl_file_info *info)
0034 {
0035     int minor;
0036
0037     mutex_lock(&minors_idr_lock);
0038     minor = idr_alloc(&minors_idr, info, 0, OCXL_NUM_MINORS, GFP_KERNEL);
0039     mutex_unlock(&minors_idr_lock);
0040     return minor;
0041 }
0042
0043 static void free_minor(struct ocxl_file_info *info)
0044 {
0045     mutex_lock(&minors_idr_lock);
0046     idr_remove(&minors_idr, MINOR(info->dev.devt));
0047     mutex_unlock(&minors_idr_lock);
0048 }
0049
0050 static int afu_open(struct inode *inode, struct file *file)
0051 {
0052     struct ocxl_file_info *info;
0053     struct ocxl_context *ctx;
0054     int rc;
0055
0056     pr_debug("%s for device %x\n", __func__, inode->i_rdev);
0057
0058     info = find_and_get_file_info(inode->i_rdev);
0059     if (!info)
0060         return -ENODEV;
0061
0062     rc = ocxl_context_alloc(&ctx, info->afu, inode->i_mapping);
0063     if (rc) {
0064         put_device(&info->dev);
0065         return rc;
0066     }
0067     put_device(&info->dev);
0068     file->private_data = ctx;
0069     return 0;
0070 }
0071
0072 static long afu_ioctl_attach(struct ocxl_context *ctx,
0073             struct ocxl_ioctl_attach __user *uarg)
0074 {
0075     struct ocxl_ioctl_attach arg;
0076     u64 amr = 0;
0077
0078     pr_debug("%s for context %d\n", __func__, ctx->pasid);
0079
0080     if (copy_from_user(&arg, uarg, sizeof(arg)))
0081         return -EFAULT;
0082
0083     /* Make sure reserved fields are not set for forward compatibility */
0084     if (arg.reserved1 || arg.reserved2 || arg.reserved3)
0085         return -EINVAL;
0086
0087     amr = arg.amr & mfspr(SPRN_UAMOR);
0088     return ocxl_context_attach(ctx, amr, current->mm);
0089 }
0090
0091 static long afu_ioctl_get_metadata(struct ocxl_context *ctx,
0092         struct ocxl_ioctl_metadata __user *uarg)
0093 {
0094     struct ocxl_ioctl_metadata arg;
0095
0096     memset(&arg, 0, sizeof(arg));
0097
0098     arg.version = 0;
0099
0100     arg.afu_version_major = ctx->afu->config.version_major;
0101     arg.afu_version_minor = ctx->afu->config.version_minor;
0102     arg.pasid = ctx->pasid;
0103     arg.pp_mmio_size = ctx->afu->config.pp_mmio_stride;
0104     arg.global_mmio_size = ctx->afu->config.global_mmio_size;
0105
0106     if (copy_to_user(uarg, &arg, sizeof(arg)))
0107         return -EFAULT;
0108
0109     return 0;
0110 }
0111
0112 #ifdef CONFIG_PPC64
0113 static long afu_ioctl_enable_p9_wait(struct ocxl_context *ctx,
0114         struct ocxl_ioctl_p9_wait __user *uarg)
0115 {
0116     struct ocxl_ioctl_p9_wait arg;
0117
0118     memset(&arg, 0, sizeof(arg));
0119
0120     if (cpu_has_feature(CPU_FTR_P9_TIDR)) {
0121         enum ocxl_context_status status;
0122
0123         // Locks both status & tidr
0124         mutex_lock(&ctx->status_mutex);
0125         if (!ctx->tidr) {
0126             if (set_thread_tidr(current)) {
0127                 mutex_unlock(&ctx->status_mutex);
0128                 return -ENOENT;
0129             }
0130
0131             ctx->tidr = current->thread.tidr;
0132         }
0133
0134         status = ctx->status;
0135         mutex_unlock(&ctx->status_mutex);
0136
0137         if (status == ATTACHED) {
0138             int rc = ocxl_link_update_pe(ctx->afu->fn->link,
0139                 ctx->pasid, ctx->tidr);
0140
0141             if (rc)
0142                 return rc;
0143         }
0144
0145         arg.thread_id = ctx->tidr;
0146     } else
0147         return -ENOENT;
0148
0149     if (copy_to_user(uarg, &arg, sizeof(arg)))
0150         return -EFAULT;
0151
0152     return 0;
0153 }
0154 #endif
0155
0156
0157 static long afu_ioctl_get_features(struct ocxl_context *ctx,
0158         struct ocxl_ioctl_features __user *uarg)
0159 {
0160     struct ocxl_ioctl_features arg;
0161
0162     memset(&arg, 0, sizeof(arg));
0163
0164 #ifdef CONFIG_PPC64
0165     if (cpu_has_feature(CPU_FTR_P9_TIDR))
0166         arg.flags[0] |= OCXL_IOCTL_FEATURES_FLAGS0_P9_WAIT;
0167 #endif
0168
0169     if (copy_to_user(uarg, &arg, sizeof(arg)))
0170         return -EFAULT;
0171
0172     return 0;
0173 }
0174
0175 #define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" :         \
0176             x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" :   \
0177             x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" :     \
0178             x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" : \
0179             x == OCXL_IOCTL_GET_METADATA ? "GET_METADATA" : \
0180             x == OCXL_IOCTL_ENABLE_P9_WAIT ? "ENABLE_P9_WAIT" : \
0181             x == OCXL_IOCTL_GET_FEATURES ? "GET_FEATURES" : \
0182             "UNKNOWN")
0183
0184 static irqreturn_t irq_handler(void *private)
0185 {
0186     struct eventfd_ctx *ev_ctx = private;
0187
0188     eventfd_signal(ev_ctx, 1);
0189     return IRQ_HANDLED;
0190 }
0191
0192 static void irq_free(void *private)
0193 {
0194     struct eventfd_ctx *ev_ctx = private;
0195
0196     eventfd_ctx_put(ev_ctx);
0197 }
0198
0199 static long afu_ioctl(struct file *file, unsigned int cmd,
0200         unsigned long args)
0201 {
0202     struct ocxl_context *ctx = file->private_data;
0203     struct ocxl_ioctl_irq_fd irq_fd;
0204     struct eventfd_ctx *ev_ctx;
0205     int irq_id;
0206     u64 irq_offset;
0207     long rc;
0208     bool closed;
0209
0210     pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid,
0211         CMD_STR(cmd));
0212
0213     mutex_lock(&ctx->status_mutex);
0214     closed = (ctx->status == CLOSED);
0215     mutex_unlock(&ctx->status_mutex);
0216
0217     if (closed)
0218         return -EIO;
0219
0220     switch (cmd) {
0221     case OCXL_IOCTL_ATTACH:
0222         rc = afu_ioctl_attach(ctx,
0223                 (struct ocxl_ioctl_attach __user *) args);
0224         break;
0225
0226     case OCXL_IOCTL_IRQ_ALLOC:
0227         rc = ocxl_afu_irq_alloc(ctx, &irq_id);
0228         if (!rc) {
0229             irq_offset = ocxl_irq_id_to_offset(ctx, irq_id);
0230             rc = copy_to_user((u64 __user *) args, &irq_offset,
0231                     sizeof(irq_offset));
0232             if (rc) {
0233                 ocxl_afu_irq_free(ctx, irq_id);
0234                 return -EFAULT;
0235             }
0236         }
0237         break;
0238
0239     case OCXL_IOCTL_IRQ_FREE:
0240         rc = copy_from_user(&irq_offset, (u64 __user *) args,
0241                 sizeof(irq_offset));
0242         if (rc)
0243             return -EFAULT;
0244         irq_id = ocxl_irq_offset_to_id(ctx, irq_offset);
0245         rc = ocxl_afu_irq_free(ctx, irq_id);
0246         break;
0247
0248     case OCXL_IOCTL_IRQ_SET_FD:
0249         rc = copy_from_user(&irq_fd, (u64 __user *) args,
0250                 sizeof(irq_fd));
0251         if (rc)
0252             return -EFAULT;
0253         if (irq_fd.reserved)
0254             return -EINVAL;
0255         irq_id = ocxl_irq_offset_to_id(ctx, irq_fd.irq_offset);
0256         ev_ctx = eventfd_ctx_fdget(irq_fd.eventfd);
0257         if (IS_ERR(ev_ctx))
0258             return PTR_ERR(ev_ctx);
0259         rc = ocxl_irq_set_handler(ctx, irq_id, irq_handler, irq_free, ev_ctx);
0260         break;
0261
0262     case OCXL_IOCTL_GET_METADATA:
0263         rc = afu_ioctl_get_metadata(ctx,
0264                 (struct ocxl_ioctl_metadata __user *) args);
0265         break;
0266
0267 #ifdef CONFIG_PPC64
0268     case OCXL_IOCTL_ENABLE_P9_WAIT:
0269         rc = afu_ioctl_enable_p9_wait(ctx,
0270                 (struct ocxl_ioctl_p9_wait __user *) args);
0271         break;
0272 #endif
0273
0274     case OCXL_IOCTL_GET_FEATURES:
0275         rc = afu_ioctl_get_features(ctx,
0276                 (struct ocxl_ioctl_features __user *) args);
0277         break;
0278
0279     default:
0280         rc = -EINVAL;
0281     }
0282     return rc;
0283 }
0284
0285 static long afu_compat_ioctl(struct file *file, unsigned int cmd,
0286             unsigned long args)
0287 {
0288     return afu_ioctl(file, cmd, args);
0289 }
0290
0291 static int afu_mmap(struct file *file, struct vm_area_struct *vma)
0292 {
0293     struct ocxl_context *ctx = file->private_data;
0294
0295     pr_debug("%s for context %d\n", __func__, ctx->pasid);
0296     return ocxl_context_mmap(ctx, vma);
0297 }
0298
0299 static bool has_xsl_error(struct ocxl_context *ctx)
0300 {
0301     bool ret;
0302
0303     mutex_lock(&ctx->xsl_error_lock);
0304     ret = !!ctx->xsl_error.addr;
0305     mutex_unlock(&ctx->xsl_error_lock);
0306
0307     return ret;
0308 }
0309
0310 /*
0311  * Are there any events pending on the AFU
0312  * ctx: The AFU context
0313  * Returns: true if there are events pending
0314  */
0315 static bool afu_events_pending(struct ocxl_context *ctx)
0316 {
0317     if (has_xsl_error(ctx))
0318         return true;
0319     return false;
0320 }
0321
0322 static unsigned int afu_poll(struct file *file, struct poll_table_struct *wait)
0323 {
0324     struct ocxl_context *ctx = file->private_data;
0325     unsigned int mask = 0;
0326     bool closed;
0327
0328     pr_debug("%s for context %d\n", __func__, ctx->pasid);
0329
0330     poll_wait(file, &ctx->events_wq, wait);
0331
0332     mutex_lock(&ctx->status_mutex);
0333     closed = (ctx->status == CLOSED);
0334     mutex_unlock(&ctx->status_mutex);
0335
0336     if (afu_events_pending(ctx))
0337         mask = EPOLLIN | EPOLLRDNORM;
0338     else if (closed)
0339         mask = EPOLLERR;
0340
0341     return mask;
0342 }
0343
0344 /*
0345  * Populate the supplied buffer with a single XSL error
0346  * ctx: The AFU context to report the error from
0347  * header: the event header to populate
0348  * buf: The buffer to write the body into (should be at least
0349  *      AFU_EVENT_BODY_XSL_ERROR_SIZE)
0350  * Return: the amount of buffer that was populated
0351  */
0352 static ssize_t append_xsl_error(struct ocxl_context *ctx,
0353                 struct ocxl_kernel_event_header *header,
0354                 char __user *buf)
0355 {
0356     struct ocxl_kernel_event_xsl_fault_error body;
0357
0358     memset(&body, 0, sizeof(body));
0359
0360     mutex_lock(&ctx->xsl_error_lock);
0361     if (!ctx->xsl_error.addr) {
0362         mutex_unlock(&ctx->xsl_error_lock);
0363         return 0;
0364     }
0365
0366     body.addr = ctx->xsl_error.addr;
0367     body.dsisr = ctx->xsl_error.dsisr;
0368     body.count = ctx->xsl_error.count;
0369
0370     ctx->xsl_error.addr = 0;
0371     ctx->xsl_error.dsisr = 0;
0372     ctx->xsl_error.count = 0;
0373
0374     mutex_unlock(&ctx->xsl_error_lock);
0375
0376     header->type = OCXL_AFU_EVENT_XSL_FAULT_ERROR;
0377
0378     if (copy_to_user(buf, &body, sizeof(body)))
0379         return -EFAULT;
0380
0381     return sizeof(body);
0382 }
0383
0384 #define AFU_EVENT_BODY_MAX_SIZE sizeof(struct ocxl_kernel_event_xsl_fault_error)
0385
0386 /*
0387  * Reports events on the AFU
0388  * Format:
0389  *  Header (struct ocxl_kernel_event_header)
0390  *  Body (struct ocxl_kernel_event_*)
0391  *  Header...
0392  */
0393 static ssize_t afu_read(struct file *file, char __user *buf, size_t count,
0394             loff_t *off)
0395 {
0396     struct ocxl_context *ctx = file->private_data;
0397     struct ocxl_kernel_event_header header;
0398     ssize_t rc;
0399     ssize_t used = 0;
0400     DEFINE_WAIT(event_wait);
0401
0402     memset(&header, 0, sizeof(header));
0403
0404     /* Require offset to be 0 */
0405     if (*off != 0)
0406         return -EINVAL;
0407
0408     if (count < (sizeof(struct ocxl_kernel_event_header) +
0409             AFU_EVENT_BODY_MAX_SIZE))
0410         return -EINVAL;
0411
0412     for (;;) {
0413         prepare_to_wait(&ctx->events_wq, &event_wait,
0414                 TASK_INTERRUPTIBLE);
0415
0416         if (afu_events_pending(ctx))
0417             break;
0418
0419         if (ctx->status == CLOSED)
0420             break;
0421
0422         if (file->f_flags & O_NONBLOCK) {
0423             finish_wait(&ctx->events_wq, &event_wait);
0424             return -EAGAIN;
0425         }
0426
0427         if (signal_pending(current)) {
0428             finish_wait(&ctx->events_wq, &event_wait);
0429             return -ERESTARTSYS;
0430         }
0431
0432         schedule();
0433     }
0434
0435     finish_wait(&ctx->events_wq, &event_wait);
0436
0437     if (has_xsl_error(ctx)) {
0438         used = append_xsl_error(ctx, &header, buf + sizeof(header));
0439         if (used < 0)
0440             return used;
0441     }
0442
0443     if (!afu_events_pending(ctx))
0444         header.flags |= OCXL_KERNEL_EVENT_FLAG_LAST;
0445
0446     if (copy_to_user(buf, &header, sizeof(header)))
0447         return -EFAULT;
0448
0449     used += sizeof(header);
0450
0451     rc = used;
0452     return rc;
0453 }
0454
0455 static int afu_release(struct inode *inode, struct file *file)
0456 {
0457     struct ocxl_context *ctx = file->private_data;
0458     int rc;
0459
0460     pr_debug("%s for device %x\n", __func__, inode->i_rdev);
0461     rc = ocxl_context_detach(ctx);
0462     mutex_lock(&ctx->mapping_lock);
0463     ctx->mapping = NULL;
0464     mutex_unlock(&ctx->mapping_lock);
0465     wake_up_all(&ctx->events_wq);
0466     if (rc != -EBUSY)
0467         ocxl_context_free(ctx);
0468     return 0;
0469 }
0470
0471 static const struct file_operations ocxl_afu_fops = {
0472     .owner      = THIS_MODULE,
0473     .open           = afu_open,
0474     .unlocked_ioctl = afu_ioctl,
0475     .compat_ioctl   = afu_compat_ioctl,
0476     .mmap           = afu_mmap,
0477     .poll           = afu_poll,
0478     .read           = afu_read,
0479     .release        = afu_release,
0480 };
0481
0482 // Free the info struct
0483 static void info_release(struct device *dev)
0484 {
0485     struct ocxl_file_info *info = container_of(dev, struct ocxl_file_info, dev);
0486
0487     ocxl_afu_put(info->afu);
0488     kfree(info);
0489 }
0490
0491 static int ocxl_file_make_visible(struct ocxl_file_info *info)
0492 {
0493     int rc;
0494
0495     cdev_init(&info->cdev, &ocxl_afu_fops);
0496     rc = cdev_add(&info->cdev, info->dev.devt, 1);
0497     if (rc) {
0498         dev_err(&info->dev, "Unable to add afu char device: %d\n", rc);
0499         return rc;
0500     }
0501
0502     return 0;
0503 }
0504
0505 static void ocxl_file_make_invisible(struct ocxl_file_info *info)
0506 {
0507     cdev_del(&info->cdev);
0508 }
0509
0510 int ocxl_file_register_afu(struct ocxl_afu *afu)
0511 {
0512     int minor;
0513     int rc;
0514     struct ocxl_file_info *info;
0515     struct ocxl_fn *fn = afu->fn;
0516     struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
0517
0518     info = kzalloc(sizeof(*info), GFP_KERNEL);
0519     if (info == NULL)
0520         return -ENOMEM;
0521
0522     minor = allocate_minor(info);
0523     if (minor < 0) {
0524         kfree(info);
0525         return minor;
0526     }
0527
0528     info->dev.parent = &fn->dev;
0529     info->dev.devt = MKDEV(MAJOR(ocxl_dev), minor);
0530     info->dev.class = ocxl_class;
0531     info->dev.release = info_release;
0532
0533     info->afu = afu;
0534     ocxl_afu_get(afu);
0535
0536     rc = dev_set_name(&info->dev, "%s.%s.%hhu",
0537         afu->config.name, dev_name(&pci_dev->dev), afu->config.idx);
0538     if (rc)
0539         goto err_put;
0540
0541     rc = device_register(&info->dev);
0542     if (rc)
0543         goto err_put;
0544
0545     rc = ocxl_sysfs_register_afu(info);
0546     if (rc)
0547         goto err_unregister;
0548
0549     rc = ocxl_file_make_visible(info);
0550     if (rc)
0551         goto err_unregister;
0552
0553     ocxl_afu_set_private(afu, info);
0554
0555     return 0;
0556
0557 err_unregister:
0558     ocxl_sysfs_unregister_afu(info); // safe to call even if register failed
0559     free_minor(info);
0560     device_unregister(&info->dev);
0561     return rc;
0562 err_put:
0563     ocxl_afu_put(afu);
0564     free_minor(info);
0565     kfree(info);
0566     return rc;
0567 }
0568
0569 void ocxl_file_unregister_afu(struct ocxl_afu *afu)
0570 {
0571     struct ocxl_file_info *info = ocxl_afu_get_private(afu);
0572
0573     if (!info)
0574         return;
0575
0576     ocxl_file_make_invisible(info);
0577     ocxl_sysfs_unregister_afu(info);
0578     free_minor(info);
0579     device_unregister(&info->dev);
0580 }
0581
0582 static char *ocxl_devnode(struct device *dev, umode_t *mode)
0583 {
0584     return kasprintf(GFP_KERNEL, "ocxl/%s", dev_name(dev));
0585 }
0586
0587 int ocxl_file_init(void)
0588 {
0589     int rc;
0590
0591     idr_init(&minors_idr);
0592
0593     rc = alloc_chrdev_region(&ocxl_dev, 0, OCXL_NUM_MINORS, "ocxl");
0594     if (rc) {
0595         pr_err("Unable to allocate ocxl major number: %d\n", rc);
0596         return rc;
0597     }
0598
0599     ocxl_class = class_create(THIS_MODULE, "ocxl");
0600     if (IS_ERR(ocxl_class)) {
0601         pr_err("Unable to create ocxl class\n");
0602         unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS);
0603         return PTR_ERR(ocxl_class);
0604     }
0605
0606     ocxl_class->devnode = ocxl_devnode;
0607     return 0;
0608 }
0609
0610 void ocxl_file_exit(void)
0611 {
0612     class_destroy(ocxl_class);
0613     unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS);
0614     idr_destroy(&minors_idr);
0615 }