OSCL-LXR linux-6.0.1/​drivers/​iommu/​io-pgfault.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Handle device page faults
  *
  * Copyright (C) 2020 ARM Ltd.
  */
 
 #include <linux/iommu.h>
 #include <linux/list.h>
 #include <linux/sched/mm.h>
 #include <linux/slab.h>
 #include <linux/workqueue.h>
 
 #include "iommu-sva-lib.h"
 
 /**
  * struct iopf_queue - IO Page Fault queue
  * @wq: the fault workqueue
  * @devices: devices attached to this queue
  * @lock: protects the device list
  */
 struct iopf_queue {
     struct workqueue_struct     *wq;
     struct list_head        devices;
     struct mutex            lock;
 };
 
 /**
  * struct iopf_device_param - IO Page Fault data attached to a device
  * @dev: the device that owns this param
  * @queue: IOPF queue
  * @queue_list: index into queue->devices
  * @partial: faults that are part of a Page Request Group for which the last
  *           request hasn't been submitted yet.
  */
 struct iopf_device_param {
     struct device           *dev;
     struct iopf_queue       *queue;
     struct list_head        queue_list;
     struct list_head        partial;
 };
 
 struct iopf_fault {
     struct iommu_fault      fault;
     struct list_head        list;
 };
 
 struct iopf_group {
     struct iopf_fault       last_fault;
     struct list_head        faults;
     struct work_struct      work;
     struct device           *dev;
 };
 
 static int iopf_complete_group(struct device *dev, struct iopf_fault *iopf,
                    enum iommu_page_response_code status)
 {
     struct iommu_page_response resp = {
         .version        = IOMMU_PAGE_RESP_VERSION_1,
         .pasid          = iopf->fault.prm.pasid,
         .grpid          = iopf->fault.prm.grpid,
         .code           = status,
     };
 
     if ((iopf->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID) &&
         (iopf->fault.prm.flags & IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID))
         resp.flags = IOMMU_PAGE_RESP_PASID_VALID;
 
     return iommu_page_response(dev, &resp);
 }
 
 static enum iommu_page_response_code
 iopf_handle_single(struct iopf_fault *iopf)
 {
     vm_fault_t ret;
     struct mm_struct *mm;
     struct vm_area_struct *vma;
     unsigned int access_flags = 0;
     unsigned int fault_flags = FAULT_FLAG_REMOTE;
     struct iommu_fault_page_request *prm = &iopf->fault.prm;
     enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID;
 
     if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID))
         return status;
 
     mm = iommu_sva_find(prm->pasid);
     if (IS_ERR_OR_NULL(mm))
         return status;
 
     mmap_read_lock(mm);
 
     vma = find_extend_vma(mm, prm->addr);
     if (!vma)
         /* Unmapped area */
         goto out_put_mm;
 
     if (prm->perm & IOMMU_FAULT_PERM_READ)
         access_flags |= VM_READ;
 
     if (prm->perm & IOMMU_FAULT_PERM_WRITE) {
         access_flags |= VM_WRITE;
         fault_flags |= FAULT_FLAG_WRITE;
     }
 
     if (prm->perm & IOMMU_FAULT_PERM_EXEC) {
         access_flags |= VM_EXEC;
         fault_flags |= FAULT_FLAG_INSTRUCTION;
     }
 
     if (!(prm->perm & IOMMU_FAULT_PERM_PRIV))
         fault_flags |= FAULT_FLAG_USER;
 
     if (access_flags & ~vma->vm_flags)
         /* Access fault */
         goto out_put_mm;
 
     ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL);
     status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID :
         IOMMU_PAGE_RESP_SUCCESS;
 
 out_put_mm:
     mmap_read_unlock(mm);
     mmput(mm);
 
     return status;
 }
 
 static void iopf_handle_group(struct work_struct *work)
 {
     struct iopf_group *group;
     struct iopf_fault *iopf, *next;
     enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS;
 
     group = container_of(work, struct iopf_group, work);
 
     list_for_each_entry_safe(iopf, next, &group->faults, list) {
         /*
          * For the moment, errors are sticky: don't handle subsequent
          * faults in the group if there is an error.
          */
         if (status == IOMMU_PAGE_RESP_SUCCESS)
             status = iopf_handle_single(iopf);
 
         if (!(iopf->fault.prm.flags &
               IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE))
             kfree(iopf);
     }
 
     iopf_complete_group(group->dev, &group->last_fault, status);
     kfree(group);
 }
 
 /**
  * iommu_queue_iopf - IO Page Fault handler
  * @fault: fault event
  * @cookie: struct device, passed to iommu_register_device_fault_handler.
  *
  * Add a fault to the device workqueue, to be handled by mm.
  *
  * This module doesn't handle PCI PASID Stop Marker; IOMMU drivers must discard
  * them before reporting faults. A PASID Stop Marker (LRW = 0b100) doesn't
  * expect a response. It may be generated when disabling a PASID (issuing a
  * PASID stop request) by some PCI devices.
  *
  * The PASID stop request is issued by the device driver before unbind(). Once
  * it completes, no page request is generated for this PASID anymore and
  * outstanding ones have been pushed to the IOMMU (as per PCIe 4.0r1.0 - 6.20.1
  * and 10.4.1.2 - Managing PASID TLP Prefix Usage). Some PCI devices will wait
  * for all outstanding page requests to come back with a response before
  * completing the PASID stop request. Others do not wait for page responses, and
  * instead issue this Stop Marker that tells us when the PASID can be
  * reallocated.
  *
  * It is safe to discard the Stop Marker because it is an optimization.
  * a. Page requests, which are posted requests, have been flushed to the IOMMU
  *    when the stop request completes.
  * b. The IOMMU driver flushes all fault queues on unbind() before freeing the
  *    PASID.
  *
  * So even though the Stop Marker might be issued by the device *after* the stop
  * request completes, outstanding faults will have been dealt with by the time
  * the PASID is freed.
  *
  * Return: 0 on success and <0 on error.
  */
 int iommu_queue_iopf(struct iommu_fault *fault, void *cookie)
 {
     int ret;
     struct iopf_group *group;
     struct iopf_fault *iopf, *next;
     struct iopf_device_param *iopf_param;
 
     struct device *dev = cookie;
     struct dev_iommu *param = dev->iommu;
 
     lockdep_assert_held(&param->lock);
 
     if (fault->type != IOMMU_FAULT_PAGE_REQ)
         /* Not a recoverable page fault */
         return -EOPNOTSUPP;
 
     /*
      * As long as we're holding param->lock, the queue can't be unlinked
      * from the device and therefore cannot disappear.
      */
     iopf_param = param->iopf_param;
     if (!iopf_param)
         return -ENODEV;
 
     if (!(fault->prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) {
         iopf = kzalloc(sizeof(*iopf), GFP_KERNEL);
         if (!iopf)
             return -ENOMEM;
 
         iopf->fault = *fault;
 
         /* Non-last request of a group. Postpone until the last one */
         list_add(&iopf->list, &iopf_param->partial);
 
         return 0;
     }
 
     group = kzalloc(sizeof(*group), GFP_KERNEL);
     if (!group) {
         /*
          * The caller will send a response to the hardware. But we do
          * need to clean up before leaving, otherwise partial faults
          * will be stuck.
          */
         ret = -ENOMEM;
         goto cleanup_partial;
     }
 
     group->dev = dev;
     group->last_fault.fault = *fault;
     INIT_LIST_HEAD(&group->faults);
     list_add(&group->last_fault.list, &group->faults);
     INIT_WORK(&group->work, iopf_handle_group);
 
     /* See if we have partial faults for this group */
     list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) {
         if (iopf->fault.prm.grpid == fault->prm.grpid)
             /* Insert *before* the last fault */
             list_move(&iopf->list, &group->faults);
     }
 
     queue_work(iopf_param->queue->wq, &group->work);
     return 0;
 
 cleanup_partial:
     list_for_each_entry_safe(iopf, next, &iopf_param->partial, list) {
         if (iopf->fault.prm.grpid == fault->prm.grpid) {
             list_del(&iopf->list);
             kfree(iopf);
         }
     }
     return ret;
 }
 EXPORT_SYMBOL_GPL(iommu_queue_iopf);
 
 /**
  * iopf_queue_flush_dev - Ensure that all queued faults have been processed
  * @dev: the endpoint whose faults need to be flushed.
  *
  * The IOMMU driver calls this before releasing a PASID, to ensure that all
  * pending faults for this PASID have been handled, and won't hit the address
  * space of the next process that uses this PASID. The driver must make sure
  * that no new fault is added to the queue. In particular it must flush its
  * low-level queue before calling this function.
  *
  * Return: 0 on success and <0 on error.
  */
 int iopf_queue_flush_dev(struct device *dev)
 {
     int ret = 0;
     struct iopf_device_param *iopf_param;
     struct dev_iommu *param = dev->iommu;
 
     if (!param)
         return -ENODEV;
 
     mutex_lock(&param->lock);
     iopf_param = param->iopf_param;
     if (iopf_param)
         flush_workqueue(iopf_param->queue->wq);
     else
         ret = -ENODEV;
     mutex_unlock(&param->lock);
 
     return ret;
 }
 EXPORT_SYMBOL_GPL(iopf_queue_flush_dev);
 
 /**
  * iopf_queue_discard_partial - Remove all pending partial fault
  * @queue: the queue whose partial faults need to be discarded
  *
  * When the hardware queue overflows, last page faults in a group may have been
  * lost and the IOMMU driver calls this to discard all partial faults. The
  * driver shouldn't be adding new faults to this queue concurrently.
  *
  * Return: 0 on success and <0 on error.
  */
 int iopf_queue_discard_partial(struct iopf_queue *queue)
 {
     struct iopf_fault *iopf, *next;
     struct iopf_device_param *iopf_param;
 
     if (!queue)
         return -EINVAL;
 
     mutex_lock(&queue->lock);
     list_for_each_entry(iopf_param, &queue->devices, queue_list) {
         list_for_each_entry_safe(iopf, next, &iopf_param->partial,
                      list) {
             list_del(&iopf->list);
             kfree(iopf);
         }
     }
     mutex_unlock(&queue->lock);
     return 0;
 }
 EXPORT_SYMBOL_GPL(iopf_queue_discard_partial);
 
 /**
  * iopf_queue_add_device - Add producer to the fault queue
  * @queue: IOPF queue
  * @dev: device to add
  *
  * Return: 0 on success and <0 on error.
  */
 int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev)
 {
     int ret = -EBUSY;
     struct iopf_device_param *iopf_param;
     struct dev_iommu *param = dev->iommu;
 
     if (!param)
         return -ENODEV;
 
     iopf_param = kzalloc(sizeof(*iopf_param), GFP_KERNEL);
     if (!iopf_param)
         return -ENOMEM;
 
     INIT_LIST_HEAD(&iopf_param->partial);
     iopf_param->queue = queue;
     iopf_param->dev = dev;
 
     mutex_lock(&queue->lock);
     mutex_lock(&param->lock);
     if (!param->iopf_param) {
         list_add(&iopf_param->queue_list, &queue->devices);
         param->iopf_param = iopf_param;
         ret = 0;
     }
     mutex_unlock(&param->lock);
     mutex_unlock(&queue->lock);
 
     if (ret)
         kfree(iopf_param);
 
     return ret;
 }
 EXPORT_SYMBOL_GPL(iopf_queue_add_device);
 
 /**
  * iopf_queue_remove_device - Remove producer from fault queue
  * @queue: IOPF queue
  * @dev: device to remove
  *
  * Caller makes sure that no more faults are reported for this device.
  *
  * Return: 0 on success and <0 on error.
  */
 int iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev)
 {
     int ret = -EINVAL;
     struct iopf_fault *iopf, *next;
     struct iopf_device_param *iopf_param;
     struct dev_iommu *param = dev->iommu;
 
     if (!param || !queue)
         return -EINVAL;
 
     mutex_lock(&queue->lock);
     mutex_lock(&param->lock);
     iopf_param = param->iopf_param;
     if (iopf_param && iopf_param->queue == queue) {
         list_del(&iopf_param->queue_list);
         param->iopf_param = NULL;
         ret = 0;
     }
     mutex_unlock(&param->lock);
     mutex_unlock(&queue->lock);
     if (ret)
         return ret;
 
     /* Just in case some faults are still stuck */
     list_for_each_entry_safe(iopf, next, &iopf_param->partial, list)
         kfree(iopf);
 
     kfree(iopf_param);
 
     return 0;
 }
 EXPORT_SYMBOL_GPL(iopf_queue_remove_device);
 
 /**
  * iopf_queue_alloc - Allocate and initialize a fault queue
  * @name: a unique string identifying the queue (for workqueue)
  *
  * Return: the queue on success and NULL on error.
  */
 struct iopf_queue *iopf_queue_alloc(const char *name)
 {
     struct iopf_queue *queue;
 
     queue = kzalloc(sizeof(*queue), GFP_KERNEL);
     if (!queue)
         return NULL;
 
     /*
      * The WQ is unordered because the low-level handler enqueues faults by
      * group. PRI requests within a group have to be ordered, but once
      * that's dealt with, the high-level function can handle groups out of
      * order.
      */
     queue->wq = alloc_workqueue("iopf_queue/%s", WQ_UNBOUND, 0, name);
     if (!queue->wq) {
         kfree(queue);
         return NULL;
     }
 
     INIT_LIST_HEAD(&queue->devices);
     mutex_init(&queue->lock);
 
     return queue;
 }
 EXPORT_SYMBOL_GPL(iopf_queue_alloc);
 
 /**
  * iopf_queue_free - Free IOPF queue
  * @queue: queue to free
  *
  * Counterpart to iopf_queue_alloc(). The driver must not be queuing faults or
  * adding/removing devices on this queue anymore.
  */
 void iopf_queue_free(struct iopf_queue *queue)
 {
     struct iopf_device_param *iopf_param, *next;
 
     if (!queue)
         return;
 
     list_for_each_entry_safe(iopf_param, next, &queue->devices, queue_list)
         iopf_queue_remove_device(queue, iopf_param->dev);
 
     destroy_workqueue(queue->wq);
     kfree(queue);
 }
 EXPORT_SYMBOL_GPL(iopf_queue_free);

This page was automatically generated by the 2.1.0 LXR engine. The LXR team