OSCL-LXR linux-6.0.1/​drivers/​iio/​buffer/​industrialio-buffer-dma.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright 2013-2015 Analog Devices Inc.
  *  Author: Lars-Peter Clausen <lars@metafoo.de>
  */
 
 #include <linux/slab.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/device.h>
 #include <linux/workqueue.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/poll.h>
 #include <linux/iio/buffer_impl.h>
 #include <linux/iio/buffer-dma.h>
 #include <linux/dma-mapping.h>
 #include <linux/sizes.h>
 
 /*
  * For DMA buffers the storage is sub-divided into so called blocks. Each block
  * has its own memory buffer. The size of the block is the granularity at which
  * memory is exchanged between the hardware and the application. Increasing the
  * basic unit of data exchange from one sample to one block decreases the
  * management overhead that is associated with each sample. E.g. if we say the
  * management overhead for one exchange is x and the unit of exchange is one
  * sample the overhead will be x for each sample. Whereas when using a block
  * which contains n samples the overhead per sample is reduced to x/n. This
  * allows to achieve much higher samplerates than what can be sustained with
  * the one sample approach.
  *
  * Blocks are exchanged between the DMA controller and the application via the
  * means of two queues. The incoming queue and the outgoing queue. Blocks on the
  * incoming queue are waiting for the DMA controller to pick them up and fill
  * them with data. Block on the outgoing queue have been filled with data and
  * are waiting for the application to dequeue them and read the data.
  *
  * A block can be in one of the following states:
  *  * Owned by the application. In this state the application can read data from
  *    the block.
  *  * On the incoming list: Blocks on the incoming list are queued up to be
  *    processed by the DMA controller.
  *  * Owned by the DMA controller: The DMA controller is processing the block
  *    and filling it with data.
  *  * On the outgoing list: Blocks on the outgoing list have been successfully
  *    processed by the DMA controller and contain data. They can be dequeued by
  *    the application.
  *  * Dead: A block that is dead has been marked as to be freed. It might still
  *    be owned by either the application or the DMA controller at the moment.
  *    But once they are done processing it instead of going to either the
  *    incoming or outgoing queue the block will be freed.
  *
  * In addition to this blocks are reference counted and the memory associated
  * with both the block structure as well as the storage memory for the block
  * will be freed when the last reference to the block is dropped. This means a
  * block must not be accessed without holding a reference.
  *
  * The iio_dma_buffer implementation provides a generic infrastructure for
  * managing the blocks.
  *
  * A driver for a specific piece of hardware that has DMA capabilities need to
  * implement the submit() callback from the iio_dma_buffer_ops structure. This
  * callback is supposed to initiate the DMA transfer copying data from the
  * converter to the memory region of the block. Once the DMA transfer has been
  * completed the driver must call iio_dma_buffer_block_done() for the completed
  * block.
  *
  * Prior to this it must set the bytes_used field of the block contains
  * the actual number of bytes in the buffer. Typically this will be equal to the
  * size of the block, but if the DMA hardware has certain alignment requirements
  * for the transfer length it might choose to use less than the full size. In
  * either case it is expected that bytes_used is a multiple of the bytes per
  * datum, i.e. the block must not contain partial samples.
  *
  * The driver must call iio_dma_buffer_block_done() for each block it has
  * received through its submit_block() callback, even if it does not actually
  * perform a DMA transfer for the block, e.g. because the buffer was disabled
  * before the block transfer was started. In this case it should set bytes_used
  * to 0.
  *
  * In addition it is recommended that a driver implements the abort() callback.
  * It will be called when the buffer is disabled and can be used to cancel
  * pending and stop active transfers.
  *
  * The specific driver implementation should use the default callback
  * implementations provided by this module for the iio_buffer_access_funcs
  * struct. It may overload some callbacks with custom variants if the hardware
  * has special requirements that are not handled by the generic functions. If a
  * driver chooses to overload a callback it has to ensure that the generic
  * callback is called from within the custom callback.
  */
 
 static void iio_buffer_block_release(struct kref *kref)
 {
     struct iio_dma_buffer_block *block = container_of(kref,
         struct iio_dma_buffer_block, kref);
 
     WARN_ON(block->state != IIO_BLOCK_STATE_DEAD);
 
     dma_free_coherent(block->queue->dev, PAGE_ALIGN(block->size),
                     block->vaddr, block->phys_addr);
 
     iio_buffer_put(&block->queue->buffer);
     kfree(block);
 }
 
 static void iio_buffer_block_get(struct iio_dma_buffer_block *block)
 {
     kref_get(&block->kref);
 }
 
 static void iio_buffer_block_put(struct iio_dma_buffer_block *block)
 {
     kref_put(&block->kref, iio_buffer_block_release);
 }
 
 /*
  * dma_free_coherent can sleep, hence we need to take some special care to be
  * able to drop a reference from an atomic context.
  */
 static LIST_HEAD(iio_dma_buffer_dead_blocks);
 static DEFINE_SPINLOCK(iio_dma_buffer_dead_blocks_lock);
 
 static void iio_dma_buffer_cleanup_worker(struct work_struct *work)
 {
     struct iio_dma_buffer_block *block, *_block;
     LIST_HEAD(block_list);
 
     spin_lock_irq(&iio_dma_buffer_dead_blocks_lock);
     list_splice_tail_init(&iio_dma_buffer_dead_blocks, &block_list);
     spin_unlock_irq(&iio_dma_buffer_dead_blocks_lock);
 
     list_for_each_entry_safe(block, _block, &block_list, head)
         iio_buffer_block_release(&block->kref);
 }
 static DECLARE_WORK(iio_dma_buffer_cleanup_work, iio_dma_buffer_cleanup_worker);
 
 static void iio_buffer_block_release_atomic(struct kref *kref)
 {
     struct iio_dma_buffer_block *block;
     unsigned long flags;
 
     block = container_of(kref, struct iio_dma_buffer_block, kref);
 
     spin_lock_irqsave(&iio_dma_buffer_dead_blocks_lock, flags);
     list_add_tail(&block->head, &iio_dma_buffer_dead_blocks);
     spin_unlock_irqrestore(&iio_dma_buffer_dead_blocks_lock, flags);
 
     schedule_work(&iio_dma_buffer_cleanup_work);
 }
 
 /*
  * Version of iio_buffer_block_put() that can be called from atomic context
  */
 static void iio_buffer_block_put_atomic(struct iio_dma_buffer_block *block)
 {
     kref_put(&block->kref, iio_buffer_block_release_atomic);
 }
 
 static struct iio_dma_buffer_queue *iio_buffer_to_queue(struct iio_buffer *buf)
 {
     return container_of(buf, struct iio_dma_buffer_queue, buffer);
 }
 
 static struct iio_dma_buffer_block *iio_dma_buffer_alloc_block(
     struct iio_dma_buffer_queue *queue, size_t size)
 {
     struct iio_dma_buffer_block *block;
 
     block = kzalloc(sizeof(*block), GFP_KERNEL);
     if (!block)
         return NULL;
 
     block->vaddr = dma_alloc_coherent(queue->dev, PAGE_ALIGN(size),
         &block->phys_addr, GFP_KERNEL);
     if (!block->vaddr) {
         kfree(block);
         return NULL;
     }
 
     block->size = size;
     block->state = IIO_BLOCK_STATE_DEQUEUED;
     block->queue = queue;
     INIT_LIST_HEAD(&block->head);
     kref_init(&block->kref);
 
     iio_buffer_get(&queue->buffer);
 
     return block;
 }
 
 static void _iio_dma_buffer_block_done(struct iio_dma_buffer_block *block)
 {
     struct iio_dma_buffer_queue *queue = block->queue;
 
     /*
      * The buffer has already been freed by the application, just drop the
      * reference.
      */
     if (block->state != IIO_BLOCK_STATE_DEAD) {
         block->state = IIO_BLOCK_STATE_DONE;
         list_add_tail(&block->head, &queue->outgoing);
     }
 }
 
 /**
  * iio_dma_buffer_block_done() - Indicate that a block has been completed
  * @block: The completed block
  *
  * Should be called when the DMA controller has finished handling the block to
  * pass back ownership of the block to the queue.
  */
 void iio_dma_buffer_block_done(struct iio_dma_buffer_block *block)
 {
     struct iio_dma_buffer_queue *queue = block->queue;
     unsigned long flags;
 
     spin_lock_irqsave(&queue->list_lock, flags);
     _iio_dma_buffer_block_done(block);
     spin_unlock_irqrestore(&queue->list_lock, flags);
 
     iio_buffer_block_put_atomic(block);
     wake_up_interruptible_poll(&queue->buffer.pollq, EPOLLIN | EPOLLRDNORM);
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_block_done);
 
 /**
  * iio_dma_buffer_block_list_abort() - Indicate that a list block has been
  *   aborted
  * @queue: Queue for which to complete blocks.
  * @list: List of aborted blocks. All blocks in this list must be from @queue.
  *
  * Typically called from the abort() callback after the DMA controller has been
  * stopped. This will set bytes_used to 0 for each block in the list and then
  * hand the blocks back to the queue.
  */
 void iio_dma_buffer_block_list_abort(struct iio_dma_buffer_queue *queue,
     struct list_head *list)
 {
     struct iio_dma_buffer_block *block, *_block;
     unsigned long flags;
 
     spin_lock_irqsave(&queue->list_lock, flags);
     list_for_each_entry_safe(block, _block, list, head) {
         list_del(&block->head);
         block->bytes_used = 0;
         _iio_dma_buffer_block_done(block);
         iio_buffer_block_put_atomic(block);
     }
     spin_unlock_irqrestore(&queue->list_lock, flags);
 
     wake_up_interruptible_poll(&queue->buffer.pollq, EPOLLIN | EPOLLRDNORM);
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_block_list_abort);
 
 static bool iio_dma_block_reusable(struct iio_dma_buffer_block *block)
 {
     /*
      * If the core owns the block it can be re-used. This should be the
      * default case when enabling the buffer, unless the DMA controller does
      * not support abort and has not given back the block yet.
      */
     switch (block->state) {
     case IIO_BLOCK_STATE_DEQUEUED:
     case IIO_BLOCK_STATE_QUEUED:
     case IIO_BLOCK_STATE_DONE:
         return true;
     default:
         return false;
     }
 }
 
 /**
  * iio_dma_buffer_request_update() - DMA buffer request_update callback
  * @buffer: The buffer which to request an update
  *
  * Should be used as the iio_dma_buffer_request_update() callback for
  * iio_buffer_access_ops struct for DMA buffers.
  */
 int iio_dma_buffer_request_update(struct iio_buffer *buffer)
 {
     struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
     struct iio_dma_buffer_block *block;
     bool try_reuse = false;
     size_t size;
     int ret = 0;
     int i;
 
     /*
      * Split the buffer into two even parts. This is used as a double
      * buffering scheme with usually one block at a time being used by the
      * DMA and the other one by the application.
      */
     size = DIV_ROUND_UP(queue->buffer.bytes_per_datum *
         queue->buffer.length, 2);
 
     mutex_lock(&queue->lock);
 
     /* Allocations are page aligned */
     if (PAGE_ALIGN(queue->fileio.block_size) == PAGE_ALIGN(size))
         try_reuse = true;
 
     queue->fileio.block_size = size;
     queue->fileio.active_block = NULL;
 
     spin_lock_irq(&queue->list_lock);
     for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) {
         block = queue->fileio.blocks[i];
 
         /* If we can't re-use it free it */
         if (block && (!iio_dma_block_reusable(block) || !try_reuse))
             block->state = IIO_BLOCK_STATE_DEAD;
     }
 
     /*
      * At this point all blocks are either owned by the core or marked as
      * dead. This means we can reset the lists without having to fear
      * corrution.
      */
     INIT_LIST_HEAD(&queue->outgoing);
     spin_unlock_irq(&queue->list_lock);
 
     INIT_LIST_HEAD(&queue->incoming);
 
     for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) {
         if (queue->fileio.blocks[i]) {
             block = queue->fileio.blocks[i];
             if (block->state == IIO_BLOCK_STATE_DEAD) {
                 /* Could not reuse it */
                 iio_buffer_block_put(block);
                 block = NULL;
             } else {
                 block->size = size;
             }
         } else {
             block = NULL;
         }
 
         if (!block) {
             block = iio_dma_buffer_alloc_block(queue, size);
             if (!block) {
                 ret = -ENOMEM;
                 goto out_unlock;
             }
             queue->fileio.blocks[i] = block;
         }
 
         block->state = IIO_BLOCK_STATE_QUEUED;
         list_add_tail(&block->head, &queue->incoming);
     }
 
 out_unlock:
     mutex_unlock(&queue->lock);
 
     return ret;
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_request_update);
 
 static void iio_dma_buffer_submit_block(struct iio_dma_buffer_queue *queue,
     struct iio_dma_buffer_block *block)
 {
     int ret;
 
     /*
      * If the hardware has already been removed we put the block into
      * limbo. It will neither be on the incoming nor outgoing list, nor will
      * it ever complete. It will just wait to be freed eventually.
      */
     if (!queue->ops)
         return;
 
     block->state = IIO_BLOCK_STATE_ACTIVE;
     iio_buffer_block_get(block);
     ret = queue->ops->submit(queue, block);
     if (ret) {
         /*
          * This is a bit of a problem and there is not much we can do
          * other then wait for the buffer to be disabled and re-enabled
          * and try again. But it should not really happen unless we run
          * out of memory or something similar.
          *
          * TODO: Implement support in the IIO core to allow buffers to
          * notify consumers that something went wrong and the buffer
          * should be disabled.
          */
         iio_buffer_block_put(block);
     }
 }
 
 /**
  * iio_dma_buffer_enable() - Enable DMA buffer
  * @buffer: IIO buffer to enable
  * @indio_dev: IIO device the buffer is attached to
  *
  * Needs to be called when the device that the buffer is attached to starts
  * sampling. Typically should be the iio_buffer_access_ops enable callback.
  *
  * This will allocate the DMA buffers and start the DMA transfers.
  */
 int iio_dma_buffer_enable(struct iio_buffer *buffer,
     struct iio_dev *indio_dev)
 {
     struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
     struct iio_dma_buffer_block *block, *_block;
 
     mutex_lock(&queue->lock);
     queue->active = true;
     list_for_each_entry_safe(block, _block, &queue->incoming, head) {
         list_del(&block->head);
         iio_dma_buffer_submit_block(queue, block);
     }
     mutex_unlock(&queue->lock);
 
     return 0;
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_enable);
 
 /**
  * iio_dma_buffer_disable() - Disable DMA buffer
  * @buffer: IIO DMA buffer to disable
  * @indio_dev: IIO device the buffer is attached to
  *
  * Needs to be called when the device that the buffer is attached to stops
  * sampling. Typically should be the iio_buffer_access_ops disable callback.
  */
 int iio_dma_buffer_disable(struct iio_buffer *buffer,
     struct iio_dev *indio_dev)
 {
     struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
 
     mutex_lock(&queue->lock);
     queue->active = false;
 
     if (queue->ops && queue->ops->abort)
         queue->ops->abort(queue);
     mutex_unlock(&queue->lock);
 
     return 0;
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_disable);
 
 static void iio_dma_buffer_enqueue(struct iio_dma_buffer_queue *queue,
     struct iio_dma_buffer_block *block)
 {
     if (block->state == IIO_BLOCK_STATE_DEAD) {
         iio_buffer_block_put(block);
     } else if (queue->active) {
         iio_dma_buffer_submit_block(queue, block);
     } else {
         block->state = IIO_BLOCK_STATE_QUEUED;
         list_add_tail(&block->head, &queue->incoming);
     }
 }
 
 static struct iio_dma_buffer_block *iio_dma_buffer_dequeue(
     struct iio_dma_buffer_queue *queue)
 {
     struct iio_dma_buffer_block *block;
 
     spin_lock_irq(&queue->list_lock);
     block = list_first_entry_or_null(&queue->outgoing, struct
         iio_dma_buffer_block, head);
     if (block != NULL) {
         list_del(&block->head);
         block->state = IIO_BLOCK_STATE_DEQUEUED;
     }
     spin_unlock_irq(&queue->list_lock);
 
     return block;
 }
 
 /**
  * iio_dma_buffer_read() - DMA buffer read callback
  * @buffer: Buffer to read form
  * @n: Number of bytes to read
  * @user_buffer: Userspace buffer to copy the data to
  *
  * Should be used as the read callback for iio_buffer_access_ops
  * struct for DMA buffers.
  */
 int iio_dma_buffer_read(struct iio_buffer *buffer, size_t n,
     char __user *user_buffer)
 {
     struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer);
     struct iio_dma_buffer_block *block;
     int ret;
 
     if (n < buffer->bytes_per_datum)
         return -EINVAL;
 
     mutex_lock(&queue->lock);
 
     if (!queue->fileio.active_block) {
         block = iio_dma_buffer_dequeue(queue);
         if (block == NULL) {
             ret = 0;
             goto out_unlock;
         }
         queue->fileio.pos = 0;
         queue->fileio.active_block = block;
     } else {
         block = queue->fileio.active_block;
     }
 
     n = rounddown(n, buffer->bytes_per_datum);
     if (n > block->bytes_used - queue->fileio.pos)
         n = block->bytes_used - queue->fileio.pos;
 
     if (copy_to_user(user_buffer, block->vaddr + queue->fileio.pos, n)) {
         ret = -EFAULT;
         goto out_unlock;
     }
 
     queue->fileio.pos += n;
 
     if (queue->fileio.pos == block->bytes_used) {
         queue->fileio.active_block = NULL;
         iio_dma_buffer_enqueue(queue, block);
     }
 
     ret = n;
 
 out_unlock:
     mutex_unlock(&queue->lock);
 
     return ret;
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_read);
 
 /**
  * iio_dma_buffer_data_available() - DMA buffer data_available callback
  * @buf: Buffer to check for data availability
  *
  * Should be used as the data_available callback for iio_buffer_access_ops
  * struct for DMA buffers.
  */
 size_t iio_dma_buffer_data_available(struct iio_buffer *buf)
 {
     struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buf);
     struct iio_dma_buffer_block *block;
     size_t data_available = 0;
 
     /*
      * For counting the available bytes we'll use the size of the block not
      * the number of actual bytes available in the block. Otherwise it is
      * possible that we end up with a value that is lower than the watermark
      * but won't increase since all blocks are in use.
      */
 
     mutex_lock(&queue->lock);
     if (queue->fileio.active_block)
         data_available += queue->fileio.active_block->size;
 
     spin_lock_irq(&queue->list_lock);
     list_for_each_entry(block, &queue->outgoing, head)
         data_available += block->size;
     spin_unlock_irq(&queue->list_lock);
     mutex_unlock(&queue->lock);
 
     return data_available;
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_data_available);
 
 /**
  * iio_dma_buffer_set_bytes_per_datum() - DMA buffer set_bytes_per_datum callback
  * @buffer: Buffer to set the bytes-per-datum for
  * @bpd: The new bytes-per-datum value
  *
  * Should be used as the set_bytes_per_datum callback for iio_buffer_access_ops
  * struct for DMA buffers.
  */
 int iio_dma_buffer_set_bytes_per_datum(struct iio_buffer *buffer, size_t bpd)
 {
     buffer->bytes_per_datum = bpd;
 
     return 0;
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_set_bytes_per_datum);
 
 /**
  * iio_dma_buffer_set_length - DMA buffer set_length callback
  * @buffer: Buffer to set the length for
  * @length: The new buffer length
  *
  * Should be used as the set_length callback for iio_buffer_access_ops
  * struct for DMA buffers.
  */
 int iio_dma_buffer_set_length(struct iio_buffer *buffer, unsigned int length)
 {
     /* Avoid an invalid state */
     if (length < 2)
         length = 2;
     buffer->length = length;
     buffer->watermark = length / 2;
 
     return 0;
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_set_length);
 
 /**
  * iio_dma_buffer_init() - Initialize DMA buffer queue
  * @queue: Buffer to initialize
  * @dev: DMA device
  * @ops: DMA buffer queue callback operations
  *
  * The DMA device will be used by the queue to do DMA memory allocations. So it
  * should refer to the device that will perform the DMA to ensure that
  * allocations are done from a memory region that can be accessed by the device.
  */
 int iio_dma_buffer_init(struct iio_dma_buffer_queue *queue,
     struct device *dev, const struct iio_dma_buffer_ops *ops)
 {
     iio_buffer_init(&queue->buffer);
     queue->buffer.length = PAGE_SIZE;
     queue->buffer.watermark = queue->buffer.length / 2;
     queue->dev = dev;
     queue->ops = ops;
 
     INIT_LIST_HEAD(&queue->incoming);
     INIT_LIST_HEAD(&queue->outgoing);
 
     mutex_init(&queue->lock);
     spin_lock_init(&queue->list_lock);
 
     return 0;
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_init);
 
 /**
  * iio_dma_buffer_exit() - Cleanup DMA buffer queue
  * @queue: Buffer to cleanup
  *
  * After this function has completed it is safe to free any resources that are
  * associated with the buffer and are accessed inside the callback operations.
  */
 void iio_dma_buffer_exit(struct iio_dma_buffer_queue *queue)
 {
     unsigned int i;
 
     mutex_lock(&queue->lock);
 
     spin_lock_irq(&queue->list_lock);
     for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) {
         if (!queue->fileio.blocks[i])
             continue;
         queue->fileio.blocks[i]->state = IIO_BLOCK_STATE_DEAD;
     }
     INIT_LIST_HEAD(&queue->outgoing);
     spin_unlock_irq(&queue->list_lock);
 
     INIT_LIST_HEAD(&queue->incoming);
 
     for (i = 0; i < ARRAY_SIZE(queue->fileio.blocks); i++) {
         if (!queue->fileio.blocks[i])
             continue;
         iio_buffer_block_put(queue->fileio.blocks[i]);
         queue->fileio.blocks[i] = NULL;
     }
     queue->fileio.active_block = NULL;
     queue->ops = NULL;
 
     mutex_unlock(&queue->lock);
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_exit);
 
 /**
  * iio_dma_buffer_release() - Release final buffer resources
  * @queue: Buffer to release
  *
  * Frees resources that can't yet be freed in iio_dma_buffer_exit(). Should be
  * called in the buffers release callback implementation right before freeing
  * the memory associated with the buffer.
  */
 void iio_dma_buffer_release(struct iio_dma_buffer_queue *queue)
 {
     mutex_destroy(&queue->lock);
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_release);
 
 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
 MODULE_DESCRIPTION("DMA buffer for the IIO framework");
 MODULE_LICENSE("GPL v2");

This page was automatically generated by the 2.1.0 LXR engine. The LXR team