OSCL-LXR linux-6.0.1/​drivers/​char/​xillybus/​xillybus_core.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * linux/drivers/misc/xillybus_core.c
  *
  * Copyright 2011 Xillybus Ltd, http://xillybus.com
  *
  * Driver for the Xillybus FPGA/host framework.
  *
  * This driver interfaces with a special IP core in an FPGA, setting up
  * a pipe between a hardware FIFO in the programmable logic and a device
  * file in the host. The number of such pipes and their attributes are
  * set up on the logic. This driver detects these automatically and
  * creates the device files accordingly.
  */
 
 #include <linux/list.h>
 #include <linux/device.h>
 #include <linux/module.h>
 #include <linux/io.h>
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
 #include <linux/sched.h>
 #include <linux/fs.h>
 #include <linux/spinlock.h>
 #include <linux/mutex.h>
 #include <linux/crc32.h>
 #include <linux/poll.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/workqueue.h>
 #include "xillybus.h"
 #include "xillybus_class.h"
 
 MODULE_DESCRIPTION("Xillybus core functions");
 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
 MODULE_ALIAS("xillybus_core");
 MODULE_LICENSE("GPL v2");
 
 /* General timeout is 100 ms, rx timeout is 10 ms */
 #define XILLY_RX_TIMEOUT (10*HZ/1000)
 #define XILLY_TIMEOUT (100*HZ/1000)
 
 #define fpga_msg_ctrl_reg              0x0008
 #define fpga_dma_control_reg           0x0020
 #define fpga_dma_bufno_reg             0x0024
 #define fpga_dma_bufaddr_lowaddr_reg   0x0028
 #define fpga_dma_bufaddr_highaddr_reg  0x002c
 #define fpga_buf_ctrl_reg              0x0030
 #define fpga_buf_offset_reg            0x0034
 #define fpga_endian_reg                0x0040
 
 #define XILLYMSG_OPCODE_RELEASEBUF 1
 #define XILLYMSG_OPCODE_QUIESCEACK 2
 #define XILLYMSG_OPCODE_FIFOEOF 3
 #define XILLYMSG_OPCODE_FATAL_ERROR 4
 #define XILLYMSG_OPCODE_NONEMPTY 5
 
 static const char xillyname[] = "xillybus";
 
 static struct workqueue_struct *xillybus_wq;
 
 /*
  * Locking scheme: Mutexes protect invocations of character device methods.
  * If both locks are taken, wr_mutex is taken first, rd_mutex second.
  *
  * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
  * buffers' end_offset fields against changes made by IRQ handler (and in
  * theory, other file request handlers, but the mutex handles that). Nothing
  * else.
  * They are held for short direct memory manipulations. Needless to say,
  * no mutex locking is allowed when a spinlock is held.
  *
  * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
  *
  * register_mutex is endpoint-specific, and is held when non-atomic
  * register operations are performed. wr_mutex and rd_mutex may be
  * held when register_mutex is taken, but none of the spinlocks. Note that
  * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
  * which are unrelated to buf_offset_reg, since they are harmless.
  *
  * Blocking on the wait queues is allowed with mutexes held, but not with
  * spinlocks.
  *
  * Only interruptible blocking is allowed on mutexes and wait queues.
  *
  * All in all, the locking order goes (with skips allowed, of course):
  * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
  */
 
 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
 {
     int opcode;
     int msg_channel, msg_bufno, msg_data, msg_dir;
 
     opcode = (buf[0] >> 24) & 0xff;
     msg_dir = buf[0] & 1;
     msg_channel = (buf[0] >> 1) & 0x7ff;
     msg_bufno = (buf[0] >> 12) & 0x3ff;
     msg_data = buf[1] & 0xfffffff;
 
     dev_warn(endpoint->dev,
          "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
          opcode, msg_channel, msg_dir, msg_bufno, msg_data);
 }
 
 /*
  * xillybus_isr assumes the interrupt is allocated exclusively to it,
  * which is the natural case MSI and several other hardware-oriented
  * interrupts. Sharing is not allowed.
  */
 
 irqreturn_t xillybus_isr(int irq, void *data)
 {
     struct xilly_endpoint *ep = data;
     u32 *buf;
     unsigned int buf_size;
     int i;
     int opcode;
     unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
     struct xilly_channel *channel;
 
     buf = ep->msgbuf_addr;
     buf_size = ep->msg_buf_size/sizeof(u32);
 
     dma_sync_single_for_cpu(ep->dev, ep->msgbuf_dma_addr,
                 ep->msg_buf_size, DMA_FROM_DEVICE);
 
     for (i = 0; i < buf_size; i += 2) {
         if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
             malformed_message(ep, &buf[i]);
             dev_warn(ep->dev,
                  "Sending a NACK on counter %x (instead of %x) on entry %d\n",
                  ((buf[i+1] >> 28) & 0xf),
                  ep->msg_counter,
                  i/2);
 
             if (++ep->failed_messages > 10) {
                 dev_err(ep->dev,
                     "Lost sync with interrupt messages. Stopping.\n");
             } else {
                 dma_sync_single_for_device(ep->dev,
                                ep->msgbuf_dma_addr,
                                ep->msg_buf_size,
                                DMA_FROM_DEVICE);
 
                 iowrite32(0x01,  /* Message NACK */
                       ep->registers + fpga_msg_ctrl_reg);
             }
             return IRQ_HANDLED;
         } else if (buf[i] & (1 << 22)) /* Last message */
             break;
     }
 
     if (i >= buf_size) {
         dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
         return IRQ_HANDLED;
     }
 
     buf_size = i + 2;
 
     for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
         opcode = (buf[i] >> 24) & 0xff;
 
         msg_dir = buf[i] & 1;
         msg_channel = (buf[i] >> 1) & 0x7ff;
         msg_bufno = (buf[i] >> 12) & 0x3ff;
         msg_data = buf[i+1] & 0xfffffff;
 
         switch (opcode) {
         case XILLYMSG_OPCODE_RELEASEBUF:
             if ((msg_channel > ep->num_channels) ||
                 (msg_channel == 0)) {
                 malformed_message(ep, &buf[i]);
                 break;
             }
 
             channel = ep->channels[msg_channel];
 
             if (msg_dir) { /* Write channel */
                 if (msg_bufno >= channel->num_wr_buffers) {
                     malformed_message(ep, &buf[i]);
                     break;
                 }
                 spin_lock(&channel->wr_spinlock);
                 channel->wr_buffers[msg_bufno]->end_offset =
                     msg_data;
                 channel->wr_fpga_buf_idx = msg_bufno;
                 channel->wr_empty = 0;
                 channel->wr_sleepy = 0;
                 spin_unlock(&channel->wr_spinlock);
 
                 wake_up_interruptible(&channel->wr_wait);
 
             } else {
                 /* Read channel */
 
                 if (msg_bufno >= channel->num_rd_buffers) {
                     malformed_message(ep, &buf[i]);
                     break;
                 }
 
                 spin_lock(&channel->rd_spinlock);
                 channel->rd_fpga_buf_idx = msg_bufno;
                 channel->rd_full = 0;
                 spin_unlock(&channel->rd_spinlock);
 
                 wake_up_interruptible(&channel->rd_wait);
                 if (!channel->rd_synchronous)
                     queue_delayed_work(
                         xillybus_wq,
                         &channel->rd_workitem,
                         XILLY_RX_TIMEOUT);
             }
 
             break;
         case XILLYMSG_OPCODE_NONEMPTY:
             if ((msg_channel > ep->num_channels) ||
                 (msg_channel == 0) || (!msg_dir) ||
                 !ep->channels[msg_channel]->wr_supports_nonempty) {
                 malformed_message(ep, &buf[i]);
                 break;
             }
 
             channel = ep->channels[msg_channel];
 
             if (msg_bufno >= channel->num_wr_buffers) {
                 malformed_message(ep, &buf[i]);
                 break;
             }
             spin_lock(&channel->wr_spinlock);
             if (msg_bufno == channel->wr_host_buf_idx)
                 channel->wr_ready = 1;
             spin_unlock(&channel->wr_spinlock);
 
             wake_up_interruptible(&channel->wr_ready_wait);
 
             break;
         case XILLYMSG_OPCODE_QUIESCEACK:
             ep->idtlen = msg_data;
             wake_up_interruptible(&ep->ep_wait);
 
             break;
         case XILLYMSG_OPCODE_FIFOEOF:
             if ((msg_channel > ep->num_channels) ||
                 (msg_channel == 0) || (!msg_dir) ||
                 !ep->channels[msg_channel]->num_wr_buffers) {
                 malformed_message(ep, &buf[i]);
                 break;
             }
             channel = ep->channels[msg_channel];
             spin_lock(&channel->wr_spinlock);
             channel->wr_eof = msg_bufno;
             channel->wr_sleepy = 0;
 
             channel->wr_hangup = channel->wr_empty &&
                 (channel->wr_host_buf_idx == msg_bufno);
 
             spin_unlock(&channel->wr_spinlock);
 
             wake_up_interruptible(&channel->wr_wait);
 
             break;
         case XILLYMSG_OPCODE_FATAL_ERROR:
             ep->fatal_error = 1;
             wake_up_interruptible(&ep->ep_wait); /* For select() */
             dev_err(ep->dev,
                 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
             break;
         default:
             malformed_message(ep, &buf[i]);
             break;
         }
     }
 
     dma_sync_single_for_device(ep->dev, ep->msgbuf_dma_addr,
                    ep->msg_buf_size, DMA_FROM_DEVICE);
 
     ep->msg_counter = (ep->msg_counter + 1) & 0xf;
     ep->failed_messages = 0;
     iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
 
     return IRQ_HANDLED;
 }
 EXPORT_SYMBOL(xillybus_isr);
 
 /*
  * A few trivial memory management functions.
  * NOTE: These functions are used only on probe and remove, and therefore
  * no locks are applied!
  */
 
 static void xillybus_autoflush(struct work_struct *work);
 
 struct xilly_alloc_state {
     void *salami;
     int left_of_salami;
     int nbuffer;
     enum dma_data_direction direction;
     u32 regdirection;
 };
 
 static void xilly_unmap(void *ptr)
 {
     struct xilly_mapping *data = ptr;
 
     dma_unmap_single(data->device, data->dma_addr,
              data->size, data->direction);
 
     kfree(ptr);
 }
 
 static int xilly_map_single(struct xilly_endpoint *ep,
                 void *ptr,
                 size_t size,
                 int direction,
                 dma_addr_t *ret_dma_handle
     )
 {
     dma_addr_t addr;
     struct xilly_mapping *this;
 
     this = kzalloc(sizeof(*this), GFP_KERNEL);
     if (!this)
         return -ENOMEM;
 
     addr = dma_map_single(ep->dev, ptr, size, direction);
 
     if (dma_mapping_error(ep->dev, addr)) {
         kfree(this);
         return -ENODEV;
     }
 
     this->device = ep->dev;
     this->dma_addr = addr;
     this->size = size;
     this->direction = direction;
 
     *ret_dma_handle = addr;
 
     return devm_add_action_or_reset(ep->dev, xilly_unmap, this);
 }
 
 static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
                  struct xilly_alloc_state *s,
                  struct xilly_buffer **buffers,
                  int bufnum, int bytebufsize)
 {
     int i, rc;
     dma_addr_t dma_addr;
     struct device *dev = ep->dev;
     struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
 
     if (buffers) { /* Not the message buffer */
         this_buffer = devm_kcalloc(dev, bufnum,
                        sizeof(struct xilly_buffer),
                        GFP_KERNEL);
         if (!this_buffer)
             return -ENOMEM;
     }
 
     for (i = 0; i < bufnum; i++) {
         /*
          * Buffers are expected in descending size order, so there
          * is either enough space for this buffer or none at all.
          */
 
         if ((s->left_of_salami < bytebufsize) &&
             (s->left_of_salami > 0)) {
             dev_err(ep->dev,
                 "Corrupt buffer allocation in IDT. Aborting.\n");
             return -ENODEV;
         }
 
         if (s->left_of_salami == 0) {
             int allocorder, allocsize;
 
             allocsize = PAGE_SIZE;
             allocorder = 0;
             while (bytebufsize > allocsize) {
                 allocsize *= 2;
                 allocorder++;
             }
 
             s->salami = (void *) devm_get_free_pages(
                 dev,
                 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
                 allocorder);
             if (!s->salami)
                 return -ENOMEM;
 
             s->left_of_salami = allocsize;
         }
 
         rc = xilly_map_single(ep, s->salami,
                       bytebufsize, s->direction,
                       &dma_addr);
         if (rc)
             return rc;
 
         iowrite32((u32) (dma_addr & 0xffffffff),
               ep->registers + fpga_dma_bufaddr_lowaddr_reg);
         iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
               ep->registers + fpga_dma_bufaddr_highaddr_reg);
 
         if (buffers) { /* Not the message buffer */
             this_buffer->addr = s->salami;
             this_buffer->dma_addr = dma_addr;
             buffers[i] = this_buffer++;
 
             iowrite32(s->regdirection | s->nbuffer++,
                   ep->registers + fpga_dma_bufno_reg);
         } else {
             ep->msgbuf_addr = s->salami;
             ep->msgbuf_dma_addr = dma_addr;
             ep->msg_buf_size = bytebufsize;
 
             iowrite32(s->regdirection,
                   ep->registers + fpga_dma_bufno_reg);
         }
 
         s->left_of_salami -= bytebufsize;
         s->salami += bytebufsize;
     }
     return 0;
 }
 
 static int xilly_setupchannels(struct xilly_endpoint *ep,
                    unsigned char *chandesc,
                    int entries)
 {
     struct device *dev = ep->dev;
     int i, entry, rc;
     struct xilly_channel *channel;
     int channelnum, bufnum, bufsize, format, is_writebuf;
     int bytebufsize;
     int synchronous, allowpartial, exclusive_open, seekable;
     int supports_nonempty;
     int msg_buf_done = 0;
 
     struct xilly_alloc_state rd_alloc = {
         .salami = NULL,
         .left_of_salami = 0,
         .nbuffer = 1,
         .direction = DMA_TO_DEVICE,
         .regdirection = 0,
     };
 
     struct xilly_alloc_state wr_alloc = {
         .salami = NULL,
         .left_of_salami = 0,
         .nbuffer = 1,
         .direction = DMA_FROM_DEVICE,
         .regdirection = 0x80000000,
     };
 
     channel = devm_kcalloc(dev, ep->num_channels,
                    sizeof(struct xilly_channel), GFP_KERNEL);
     if (!channel)
         return -ENOMEM;
 
     ep->channels = devm_kcalloc(dev, ep->num_channels + 1,
                     sizeof(struct xilly_channel *),
                     GFP_KERNEL);
     if (!ep->channels)
         return -ENOMEM;
 
     ep->channels[0] = NULL; /* Channel 0 is message buf. */
 
     /* Initialize all channels with defaults */
 
     for (i = 1; i <= ep->num_channels; i++) {
         channel->wr_buffers = NULL;
         channel->rd_buffers = NULL;
         channel->num_wr_buffers = 0;
         channel->num_rd_buffers = 0;
         channel->wr_fpga_buf_idx = -1;
         channel->wr_host_buf_idx = 0;
         channel->wr_host_buf_pos = 0;
         channel->wr_empty = 1;
         channel->wr_ready = 0;
         channel->wr_sleepy = 1;
         channel->rd_fpga_buf_idx = 0;
         channel->rd_host_buf_idx = 0;
         channel->rd_host_buf_pos = 0;
         channel->rd_full = 0;
         channel->wr_ref_count = 0;
         channel->rd_ref_count = 0;
 
         spin_lock_init(&channel->wr_spinlock);
         spin_lock_init(&channel->rd_spinlock);
         mutex_init(&channel->wr_mutex);
         mutex_init(&channel->rd_mutex);
         init_waitqueue_head(&channel->rd_wait);
         init_waitqueue_head(&channel->wr_wait);
         init_waitqueue_head(&channel->wr_ready_wait);
 
         INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
 
         channel->endpoint = ep;
         channel->chan_num = i;
 
         channel->log2_element_size = 0;
 
         ep->channels[i] = channel++;
     }
 
     for (entry = 0; entry < entries; entry++, chandesc += 4) {
         struct xilly_buffer **buffers = NULL;
 
         is_writebuf = chandesc[0] & 0x01;
         channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
         format = (chandesc[1] >> 4) & 0x03;
         allowpartial = (chandesc[1] >> 6) & 0x01;
         synchronous = (chandesc[1] >> 7) & 0x01;
         bufsize = 1 << (chandesc[2] & 0x1f);
         bufnum = 1 << (chandesc[3] & 0x0f);
         exclusive_open = (chandesc[2] >> 7) & 0x01;
         seekable = (chandesc[2] >> 6) & 0x01;
         supports_nonempty = (chandesc[2] >> 5) & 0x01;
 
         if ((channelnum > ep->num_channels) ||
             ((channelnum == 0) && !is_writebuf)) {
             dev_err(ep->dev,
                 "IDT requests channel out of range. Aborting.\n");
             return -ENODEV;
         }
 
         channel = ep->channels[channelnum]; /* NULL for msg channel */
 
         if (!is_writebuf || channelnum > 0) {
             channel->log2_element_size = ((format > 2) ?
                               2 : format);
 
             bytebufsize = bufsize *
                 (1 << channel->log2_element_size);
 
             buffers = devm_kcalloc(dev, bufnum,
                            sizeof(struct xilly_buffer *),
                            GFP_KERNEL);
             if (!buffers)
                 return -ENOMEM;
         } else {
             bytebufsize = bufsize << 2;
         }
 
         if (!is_writebuf) {
             channel->num_rd_buffers = bufnum;
             channel->rd_buf_size = bytebufsize;
             channel->rd_allow_partial = allowpartial;
             channel->rd_synchronous = synchronous;
             channel->rd_exclusive_open = exclusive_open;
             channel->seekable = seekable;
 
             channel->rd_buffers = buffers;
             rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,
                            bufnum, bytebufsize);
         } else if (channelnum > 0) {
             channel->num_wr_buffers = bufnum;
             channel->wr_buf_size = bytebufsize;
 
             channel->seekable = seekable;
             channel->wr_supports_nonempty = supports_nonempty;
 
             channel->wr_allow_partial = allowpartial;
             channel->wr_synchronous = synchronous;
             channel->wr_exclusive_open = exclusive_open;
 
             channel->wr_buffers = buffers;
             rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,
                            bufnum, bytebufsize);
         } else {
             rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,
                            bufnum, bytebufsize);
             msg_buf_done++;
         }
 
         if (rc)
             return -ENOMEM;
     }
 
     if (!msg_buf_done) {
         dev_err(ep->dev,
             "Corrupt IDT: No message buffer. Aborting.\n");
         return -ENODEV;
     }
     return 0;
 }
 
 static int xilly_scan_idt(struct xilly_endpoint *endpoint,
               struct xilly_idt_handle *idt_handle)
 {
     int count = 0;
     unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
     unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
     unsigned char *scan;
     int len;
 
     scan = idt + 1;
     idt_handle->names = scan;
 
     while ((scan <= end_of_idt) && *scan) {
         while ((scan <= end_of_idt) && *scan++)
             /* Do nothing, just scan thru string */;
         count++;
     }
 
     idt_handle->names_len = scan - idt_handle->names;
 
     scan++;
 
     if (scan > end_of_idt) {
         dev_err(endpoint->dev,
             "IDT device name list overflow. Aborting.\n");
         return -ENODEV;
     }
     idt_handle->chandesc = scan;
 
     len = endpoint->idtlen - (3 + ((int) (scan - idt)));
 
     if (len & 0x03) {
         dev_err(endpoint->dev,
             "Corrupt IDT device name list. Aborting.\n");
         return -ENODEV;
     }
 
     idt_handle->entries = len >> 2;
     endpoint->num_channels = count;
 
     return 0;
 }
 
 static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
 {
     struct xilly_channel *channel;
     unsigned char *version;
     long t;
 
     channel = endpoint->channels[1]; /* This should be generated ad-hoc */
 
     channel->wr_sleepy = 1;
 
     iowrite32(1 |
           (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
           endpoint->registers + fpga_buf_ctrl_reg);
 
     t = wait_event_interruptible_timeout(channel->wr_wait,
                          (!channel->wr_sleepy),
                          XILLY_TIMEOUT);
 
     if (t <= 0) {
         dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
 
         if (endpoint->fatal_error)
             return -EIO;
 
         return -ENODEV;
     }
 
     dma_sync_single_for_cpu(channel->endpoint->dev,
                 channel->wr_buffers[0]->dma_addr,
                 channel->wr_buf_size,
                 DMA_FROM_DEVICE);
 
     if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
         dev_err(endpoint->dev,
             "IDT length mismatch (%d != %d). Aborting.\n",
             channel->wr_buffers[0]->end_offset, endpoint->idtlen);
         return -ENODEV;
     }
 
     if (crc32_le(~0, channel->wr_buffers[0]->addr,
              endpoint->idtlen+1) != 0) {
         dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
         return -ENODEV;
     }
 
     version = channel->wr_buffers[0]->addr;
 
     /* Check version number. Reject anything above 0x82. */
     if (*version > 0x82) {
         dev_err(endpoint->dev,
             "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n",
             *version);
         return -ENODEV;
     }
 
     return 0;
 }
 
 static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
                  size_t count, loff_t *f_pos)
 {
     ssize_t rc;
     unsigned long flags;
     int bytes_done = 0;
     int no_time_left = 0;
     long deadline, left_to_sleep;
     struct xilly_channel *channel = filp->private_data;
 
     int empty, reached_eof, exhausted, ready;
     /* Initializations are there only to silence warnings */
 
     int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
     int waiting_bufidx;
 
     if (channel->endpoint->fatal_error)
         return -EIO;
 
     deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
 
     rc = mutex_lock_interruptible(&channel->wr_mutex);
     if (rc)
         return rc;
 
     while (1) { /* Note that we may drop mutex within this loop */
         int bytes_to_do = count - bytes_done;
 
         spin_lock_irqsave(&channel->wr_spinlock, flags);
 
         empty = channel->wr_empty;
         ready = !empty || channel->wr_ready;
 
         if (!empty) {
             bufidx = channel->wr_host_buf_idx;
             bufpos = channel->wr_host_buf_pos;
             howmany = ((channel->wr_buffers[bufidx]->end_offset
                     + 1) << channel->log2_element_size)
                 - bufpos;
 
             /* Update wr_host_* to its post-operation state */
             if (howmany > bytes_to_do) {
                 bufferdone = 0;
 
                 howmany = bytes_to_do;
                 channel->wr_host_buf_pos += howmany;
             } else {
                 bufferdone = 1;
 
                 channel->wr_host_buf_pos = 0;
 
                 if (bufidx == channel->wr_fpga_buf_idx) {
                     channel->wr_empty = 1;
                     channel->wr_sleepy = 1;
                     channel->wr_ready = 0;
                 }
 
                 if (bufidx >= (channel->num_wr_buffers - 1))
                     channel->wr_host_buf_idx = 0;
                 else
                     channel->wr_host_buf_idx++;
             }
         }
 
         /*
          * Marking our situation after the possible changes above,
          * for use after releasing the spinlock.
          *
          * empty = empty before change
          * exhasted = empty after possible change
          */
 
         reached_eof = channel->wr_empty &&
             (channel->wr_host_buf_idx == channel->wr_eof);
         channel->wr_hangup = reached_eof;
         exhausted = channel->wr_empty;
         waiting_bufidx = channel->wr_host_buf_idx;
 
         spin_unlock_irqrestore(&channel->wr_spinlock, flags);
 
         if (!empty) { /* Go on, now without the spinlock */
 
             if (bufpos == 0) /* Position zero means it's virgin */
                 dma_sync_single_for_cpu(channel->endpoint->dev,
                             channel->wr_buffers[bufidx]->dma_addr,
                             channel->wr_buf_size,
                             DMA_FROM_DEVICE);
 
             if (copy_to_user(
                     userbuf,
                     channel->wr_buffers[bufidx]->addr
                     + bufpos, howmany))
                 rc = -EFAULT;
 
             userbuf += howmany;
             bytes_done += howmany;
 
             if (bufferdone) {
                 dma_sync_single_for_device(channel->endpoint->dev,
                                channel->wr_buffers[bufidx]->dma_addr,
                                channel->wr_buf_size,
                                DMA_FROM_DEVICE);
 
                 /*
                  * Tell FPGA the buffer is done with. It's an
                  * atomic operation to the FPGA, so what
                  * happens with other channels doesn't matter,
                  * and the certain channel is protected with
                  * the channel-specific mutex.
                  */
 
                 iowrite32(1 | (channel->chan_num << 1) |
                       (bufidx << 12),
                       channel->endpoint->registers +
                       fpga_buf_ctrl_reg);
             }
 
             if (rc) {
                 mutex_unlock(&channel->wr_mutex);
                 return rc;
             }
         }
 
         /* This includes a zero-count return = EOF */
         if ((bytes_done >= count) || reached_eof)
             break;
 
         if (!exhausted)
             continue; /* More in RAM buffer(s)? Just go on. */
 
         if ((bytes_done > 0) &&
             (no_time_left ||
              (channel->wr_synchronous && channel->wr_allow_partial)))
             break;
 
         /*
          * Nonblocking read: The "ready" flag tells us that the FPGA
          * has data to send. In non-blocking mode, if it isn't on,
          * just return. But if there is, we jump directly to the point
          * where we ask for the FPGA to send all it has, and wait
          * until that data arrives. So in a sense, we *do* block in
          * nonblocking mode, but only for a very short time.
          */
 
         if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
             if (bytes_done > 0)
                 break;
 
             if (ready)
                 goto desperate;
 
             rc = -EAGAIN;
             break;
         }
 
         if (!no_time_left || (bytes_done > 0)) {
             /*
              * Note that in case of an element-misaligned read
              * request, offsetlimit will include the last element,
              * which will be partially read from.
              */
             int offsetlimit = ((count - bytes_done) - 1) >>
                 channel->log2_element_size;
             int buf_elements = channel->wr_buf_size >>
                 channel->log2_element_size;
 
             /*
              * In synchronous mode, always send an offset limit.
              * Just don't send a value too big.
              */
 
             if (channel->wr_synchronous) {
                 /* Don't request more than one buffer */
                 if (channel->wr_allow_partial &&
                     (offsetlimit >= buf_elements))
                     offsetlimit = buf_elements - 1;
 
                 /* Don't request more than all buffers */
                 if (!channel->wr_allow_partial &&
                     (offsetlimit >=
                      (buf_elements * channel->num_wr_buffers)))
                     offsetlimit = buf_elements *
                         channel->num_wr_buffers - 1;
             }
 
             /*
              * In asynchronous mode, force early flush of a buffer
              * only if that will allow returning a full count. The
              * "offsetlimit < ( ... )" rather than "<=" excludes
              * requesting a full buffer, which would obviously
              * cause a buffer transmission anyhow
              */
 
             if (channel->wr_synchronous ||
                 (offsetlimit < (buf_elements - 1))) {
                 mutex_lock(&channel->endpoint->register_mutex);
 
                 iowrite32(offsetlimit,
                       channel->endpoint->registers +
                       fpga_buf_offset_reg);
 
                 iowrite32(1 | (channel->chan_num << 1) |
                       (2 << 24) |  /* 2 = offset limit */
                       (waiting_bufidx << 12),
                       channel->endpoint->registers +
                       fpga_buf_ctrl_reg);
 
                 mutex_unlock(&channel->endpoint->
                          register_mutex);
             }
         }
 
         /*
          * If partial completion is disallowed, there is no point in
          * timeout sleeping. Neither if no_time_left is set and
          * there's no data.
          */
 
         if (!channel->wr_allow_partial ||
             (no_time_left && (bytes_done == 0))) {
             /*
              * This do-loop will run more than once if another
              * thread reasserted wr_sleepy before we got the mutex
              * back, so we try again.
              */
 
             do {
                 mutex_unlock(&channel->wr_mutex);
 
                 if (wait_event_interruptible(
                         channel->wr_wait,
                         (!channel->wr_sleepy)))
                     goto interrupted;
 
                 if (mutex_lock_interruptible(
                         &channel->wr_mutex))
                     goto interrupted;
             } while (channel->wr_sleepy);
 
             continue;
 
 interrupted: /* Mutex is not held if got here */
             if (channel->endpoint->fatal_error)
                 return -EIO;
             if (bytes_done)
                 return bytes_done;
             if (filp->f_flags & O_NONBLOCK)
                 return -EAGAIN; /* Don't admit snoozing */
             return -EINTR;
         }
 
         left_to_sleep = deadline - ((long) jiffies);
 
         /*
          * If our time is out, skip the waiting. We may miss wr_sleepy
          * being deasserted but hey, almost missing the train is like
          * missing it.
          */
 
         if (left_to_sleep > 0) {
             left_to_sleep =
                 wait_event_interruptible_timeout(
                     channel->wr_wait,
                     (!channel->wr_sleepy),
                     left_to_sleep);
 
             if (left_to_sleep > 0) /* wr_sleepy deasserted */
                 continue;
 
             if (left_to_sleep < 0) { /* Interrupt */
                 mutex_unlock(&channel->wr_mutex);
                 if (channel->endpoint->fatal_error)
                     return -EIO;
                 if (bytes_done)
                     return bytes_done;
                 return -EINTR;
             }
         }
 
 desperate:
         no_time_left = 1; /* We're out of sleeping time. Desperate! */
 
         if (bytes_done == 0) {
             /*
              * Reaching here means that we allow partial return,
              * that we've run out of time, and that we have
              * nothing to return.
              * So tell the FPGA to send anything it has or gets.
              */
 
             iowrite32(1 | (channel->chan_num << 1) |
                   (3 << 24) |  /* Opcode 3, flush it all! */
                   (waiting_bufidx << 12),
                   channel->endpoint->registers +
                   fpga_buf_ctrl_reg);
         }
 
         /*
          * Reaching here means that we *do* have data in the buffer,
          * but the "partial" flag disallows returning less than
          * required. And we don't have as much. So loop again,
          * which is likely to end up blocking indefinitely until
          * enough data has arrived.
          */
     }
 
     mutex_unlock(&channel->wr_mutex);
 
     if (channel->endpoint->fatal_error)
         return -EIO;
 
     if (rc)
         return rc;
 
     return bytes_done;
 }
 
 /*
  * The timeout argument takes values as follows:
  *  >0 : Flush with timeout
  * ==0 : Flush, and wait idefinitely for the flush to complete
  *  <0 : Autoflush: Flush only if there's a single buffer occupied
  */
 
 static int xillybus_myflush(struct xilly_channel *channel, long timeout)
 {
     int rc;
     unsigned long flags;
 
     int end_offset_plus1;
     int bufidx, bufidx_minus1;
     int i;
     int empty;
     int new_rd_host_buf_pos;
 
     if (channel->endpoint->fatal_error)
         return -EIO;
     rc = mutex_lock_interruptible(&channel->rd_mutex);
     if (rc)
         return rc;
 
     /*
      * Don't flush a closed channel. This can happen when the work queued
      * autoflush thread fires off after the file has closed. This is not
      * an error, just something to dismiss.
      */
 
     if (!channel->rd_ref_count)
         goto done;
 
     bufidx = channel->rd_host_buf_idx;
 
     bufidx_minus1 = (bufidx == 0) ?
         channel->num_rd_buffers - 1 :
         bufidx - 1;
 
     end_offset_plus1 = channel->rd_host_buf_pos >>
         channel->log2_element_size;
 
     new_rd_host_buf_pos = channel->rd_host_buf_pos -
         (end_offset_plus1 << channel->log2_element_size);
 
     /* Submit the current buffer if it's nonempty */
     if (end_offset_plus1) {
         unsigned char *tail = channel->rd_buffers[bufidx]->addr +
             (end_offset_plus1 << channel->log2_element_size);
 
         /* Copy  unflushed data, so we can put it in next buffer */
         for (i = 0; i < new_rd_host_buf_pos; i++)
             channel->rd_leftovers[i] = *tail++;
 
         spin_lock_irqsave(&channel->rd_spinlock, flags);
 
         /* Autoflush only if a single buffer is occupied */
 
         if ((timeout < 0) &&
             (channel->rd_full ||
              (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
             spin_unlock_irqrestore(&channel->rd_spinlock, flags);
             /*
              * A new work item may be queued by the ISR exactly
              * now, since the execution of a work item allows the
              * queuing of a new one while it's running.
              */
             goto done;
         }
 
         /* The 4th element is never needed for data, so it's a flag */
         channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
 
         /* Set up rd_full to reflect a certain moment's state */
 
         if (bufidx == channel->rd_fpga_buf_idx)
             channel->rd_full = 1;
         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
 
         if (bufidx >= (channel->num_rd_buffers - 1))
             channel->rd_host_buf_idx = 0;
         else
             channel->rd_host_buf_idx++;
 
         dma_sync_single_for_device(channel->endpoint->dev,
                        channel->rd_buffers[bufidx]->dma_addr,
                        channel->rd_buf_size,
                        DMA_TO_DEVICE);
 
         mutex_lock(&channel->endpoint->register_mutex);
 
         iowrite32(end_offset_plus1 - 1,
               channel->endpoint->registers + fpga_buf_offset_reg);
 
         iowrite32((channel->chan_num << 1) | /* Channel ID */
               (2 << 24) |  /* Opcode 2, submit buffer */
               (bufidx << 12),
               channel->endpoint->registers + fpga_buf_ctrl_reg);
 
         mutex_unlock(&channel->endpoint->register_mutex);
     } else if (bufidx == 0) {
         bufidx = channel->num_rd_buffers - 1;
     } else {
         bufidx--;
     }
 
     channel->rd_host_buf_pos = new_rd_host_buf_pos;
 
     if (timeout < 0)
         goto done; /* Autoflush */
 
     /*
      * bufidx is now the last buffer written to (or equal to
      * rd_fpga_buf_idx if buffer was never written to), and
      * channel->rd_host_buf_idx the one after it.
      *
      * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
      */
 
     while (1) { /* Loop waiting for draining of buffers */
         spin_lock_irqsave(&channel->rd_spinlock, flags);
 
         if (bufidx != channel->rd_fpga_buf_idx)
             channel->rd_full = 1; /*
                            * Not really full,
                            * but needs waiting.
                            */
 
         empty = !channel->rd_full;
 
         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
 
         if (empty)
             break;
 
         /*
          * Indefinite sleep with mutex taken. With data waiting for
          * flushing user should not be surprised if open() for write
          * sleeps.
          */
         if (timeout == 0)
             wait_event_interruptible(channel->rd_wait,
                          (!channel->rd_full));
 
         else if (wait_event_interruptible_timeout(
                  channel->rd_wait,
                  (!channel->rd_full),
                  timeout) == 0) {
             dev_warn(channel->endpoint->dev,
                  "Timed out while flushing. Output data may be lost.\n");
 
             rc = -ETIMEDOUT;
             break;
         }
 
         if (channel->rd_full) {
             rc = -EINTR;
             break;
         }
     }
 
 done:
     mutex_unlock(&channel->rd_mutex);
 
     if (channel->endpoint->fatal_error)
         return -EIO;
 
     return rc;
 }
 
 static int xillybus_flush(struct file *filp, fl_owner_t id)
 {
     if (!(filp->f_mode & FMODE_WRITE))
         return 0;
 
     return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
 }
 
 static void xillybus_autoflush(struct work_struct *work)
 {
     struct delayed_work *workitem = container_of(
         work, struct delayed_work, work);
     struct xilly_channel *channel = container_of(
         workitem, struct xilly_channel, rd_workitem);
     int rc;
 
     rc = xillybus_myflush(channel, -1);
     if (rc == -EINTR)
         dev_warn(channel->endpoint->dev,
              "Autoflush failed because work queue thread got a signal.\n");
     else if (rc)
         dev_err(channel->endpoint->dev,
             "Autoflush failed under weird circumstances.\n");
 }
 
 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
                   size_t count, loff_t *f_pos)
 {
     ssize_t rc;
     unsigned long flags;
     int bytes_done = 0;
     struct xilly_channel *channel = filp->private_data;
 
     int full, exhausted;
     /* Initializations are there only to silence warnings */
 
     int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
     int end_offset_plus1 = 0;
 
     if (channel->endpoint->fatal_error)
         return -EIO;
 
     rc = mutex_lock_interruptible(&channel->rd_mutex);
     if (rc)
         return rc;
 
     while (1) {
         int bytes_to_do = count - bytes_done;
 
         spin_lock_irqsave(&channel->rd_spinlock, flags);
 
         full = channel->rd_full;
 
         if (!full) {
             bufidx = channel->rd_host_buf_idx;
             bufpos = channel->rd_host_buf_pos;
             howmany = channel->rd_buf_size - bufpos;
 
             /*
              * Update rd_host_* to its state after this operation.
              * count=0 means committing the buffer immediately,
              * which is like flushing, but not necessarily block.
              */
 
             if ((howmany > bytes_to_do) &&
                 (count ||
                  ((bufpos >> channel->log2_element_size) == 0))) {
                 bufferdone = 0;
 
                 howmany = bytes_to_do;
                 channel->rd_host_buf_pos += howmany;
             } else {
                 bufferdone = 1;
 
                 if (count) {
                     end_offset_plus1 =
                         channel->rd_buf_size >>
                         channel->log2_element_size;
                     channel->rd_host_buf_pos = 0;
                 } else {
                     unsigned char *tail;
                     int i;
 
                     howmany = 0;
 
                     end_offset_plus1 = bufpos >>
                         channel->log2_element_size;
 
                     channel->rd_host_buf_pos -=
                         end_offset_plus1 <<
                         channel->log2_element_size;
 
                     tail = channel->
                         rd_buffers[bufidx]->addr +
                         (end_offset_plus1 <<
                          channel->log2_element_size);
 
                     for (i = 0;
                          i < channel->rd_host_buf_pos;
                          i++)
                         channel->rd_leftovers[i] =
                             *tail++;
                 }
 
                 if (bufidx == channel->rd_fpga_buf_idx)
                     channel->rd_full = 1;
 
                 if (bufidx >= (channel->num_rd_buffers - 1))
                     channel->rd_host_buf_idx = 0;
                 else
                     channel->rd_host_buf_idx++;
             }
         }
 
         /*
          * Marking our situation after the possible changes above,
          * for use  after releasing the spinlock.
          *
          * full = full before change
          * exhasted = full after possible change
          */
 
         exhausted = channel->rd_full;
 
         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
 
         if (!full) { /* Go on, now without the spinlock */
             unsigned char *head =
                 channel->rd_buffers[bufidx]->addr;
             int i;
 
             if ((bufpos == 0) || /* Zero means it's virgin */
                 (channel->rd_leftovers[3] != 0)) {
                 dma_sync_single_for_cpu(channel->endpoint->dev,
                             channel->rd_buffers[bufidx]->dma_addr,
                             channel->rd_buf_size,
                             DMA_TO_DEVICE);
 
                 /* Virgin, but leftovers are due */
                 for (i = 0; i < bufpos; i++)
                     *head++ = channel->rd_leftovers[i];
 
                 channel->rd_leftovers[3] = 0; /* Clear flag */
             }
 
             if (copy_from_user(
                     channel->rd_buffers[bufidx]->addr + bufpos,
                     userbuf, howmany))
                 rc = -EFAULT;
 
             userbuf += howmany;
             bytes_done += howmany;
 
             if (bufferdone) {
                 dma_sync_single_for_device(channel->endpoint->dev,
                                channel->rd_buffers[bufidx]->dma_addr,
                                channel->rd_buf_size,
                                DMA_TO_DEVICE);
 
                 mutex_lock(&channel->endpoint->register_mutex);
 
                 iowrite32(end_offset_plus1 - 1,
                       channel->endpoint->registers +
                       fpga_buf_offset_reg);
 
                 iowrite32((channel->chan_num << 1) |
                       (2 << 24) |  /* 2 = submit buffer */
                       (bufidx << 12),
                       channel->endpoint->registers +
                       fpga_buf_ctrl_reg);
 
                 mutex_unlock(&channel->endpoint->
                          register_mutex);
 
                 channel->rd_leftovers[3] =
                     (channel->rd_host_buf_pos != 0);
             }
 
             if (rc) {
                 mutex_unlock(&channel->rd_mutex);
 
                 if (channel->endpoint->fatal_error)
                     return -EIO;
 
                 if (!channel->rd_synchronous)
                     queue_delayed_work(
                         xillybus_wq,
                         &channel->rd_workitem,
                         XILLY_RX_TIMEOUT);
 
                 return rc;
             }
         }
 
         if (bytes_done >= count)
             break;
 
         if (!exhausted)
             continue; /* If there's more space, just go on */
 
         if ((bytes_done > 0) && channel->rd_allow_partial)
             break;
 
         /*
          * Indefinite sleep with mutex taken. With data waiting for
          * flushing, user should not be surprised if open() for write
          * sleeps.
          */
 
         if (filp->f_flags & O_NONBLOCK) {
             rc = -EAGAIN;
             break;
         }
 
         if (wait_event_interruptible(channel->rd_wait,
                          (!channel->rd_full))) {
             mutex_unlock(&channel->rd_mutex);
 
             if (channel->endpoint->fatal_error)
                 return -EIO;
 
             if (bytes_done)
                 return bytes_done;
             return -EINTR;
         }
     }
 
     mutex_unlock(&channel->rd_mutex);
 
     if (!channel->rd_synchronous)
         queue_delayed_work(xillybus_wq,
                    &channel->rd_workitem,
                    XILLY_RX_TIMEOUT);
 
     if (channel->endpoint->fatal_error)
         return -EIO;
 
     if (rc)
         return rc;
 
     if ((channel->rd_synchronous) && (bytes_done > 0)) {
         rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
 
         if (rc && (rc != -EINTR))
             return rc;
     }
 
     return bytes_done;
 }
 
 static int xillybus_open(struct inode *inode, struct file *filp)
 {
     int rc;
     unsigned long flags;
     struct xilly_endpoint *endpoint;
     struct xilly_channel *channel;
     int index;
 
     rc = xillybus_find_inode(inode, (void **)&endpoint, &index);
     if (rc)
         return rc;
 
     if (endpoint->fatal_error)
         return -EIO;
 
     channel = endpoint->channels[1 + index];
     filp->private_data = channel;
 
     /*
      * It gets complicated because:
      * 1. We don't want to take a mutex we don't have to
      * 2. We don't want to open one direction if the other will fail.
      */
 
     if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
         return -ENODEV;
 
     if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
         return -ENODEV;
 
     if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
         (channel->wr_synchronous || !channel->wr_allow_partial ||
          !channel->wr_supports_nonempty)) {
         dev_err(endpoint->dev,
             "open() failed: O_NONBLOCK not allowed for read on this device\n");
         return -ENODEV;
     }
 
     if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
         (channel->rd_synchronous || !channel->rd_allow_partial)) {
         dev_err(endpoint->dev,
             "open() failed: O_NONBLOCK not allowed for write on this device\n");
         return -ENODEV;
     }
 
     /*
      * Note: open() may block on getting mutexes despite O_NONBLOCK.
      * This shouldn't occur normally, since multiple open of the same
      * file descriptor is almost always prohibited anyhow
      * (*_exclusive_open is normally set in real-life systems).
      */
 
     if (filp->f_mode & FMODE_READ) {
         rc = mutex_lock_interruptible(&channel->wr_mutex);
         if (rc)
             return rc;
     }
 
     if (filp->f_mode & FMODE_WRITE) {
         rc = mutex_lock_interruptible(&channel->rd_mutex);
         if (rc)
             goto unlock_wr;
     }
 
     if ((filp->f_mode & FMODE_READ) &&
         (channel->wr_ref_count != 0) &&
         (channel->wr_exclusive_open)) {
         rc = -EBUSY;
         goto unlock;
     }
 
     if ((filp->f_mode & FMODE_WRITE) &&
         (channel->rd_ref_count != 0) &&
         (channel->rd_exclusive_open)) {
         rc = -EBUSY;
         goto unlock;
     }
 
     if (filp->f_mode & FMODE_READ) {
         if (channel->wr_ref_count == 0) { /* First open of file */
             /* Move the host to first buffer */
             spin_lock_irqsave(&channel->wr_spinlock, flags);
             channel->wr_host_buf_idx = 0;
             channel->wr_host_buf_pos = 0;
             channel->wr_fpga_buf_idx = -1;
             channel->wr_empty = 1;
             channel->wr_ready = 0;
             channel->wr_sleepy = 1;
             channel->wr_eof = -1;
             channel->wr_hangup = 0;
 
             spin_unlock_irqrestore(&channel->wr_spinlock, flags);
 
             iowrite32(1 | (channel->chan_num << 1) |
                   (4 << 24) |  /* Opcode 4, open channel */
                   ((channel->wr_synchronous & 1) << 23),
                   channel->endpoint->registers +
                   fpga_buf_ctrl_reg);
         }
 
         channel->wr_ref_count++;
     }
 
     if (filp->f_mode & FMODE_WRITE) {
         if (channel->rd_ref_count == 0) { /* First open of file */
             /* Move the host to first buffer */
             spin_lock_irqsave(&channel->rd_spinlock, flags);
             channel->rd_host_buf_idx = 0;
             channel->rd_host_buf_pos = 0;
             channel->rd_leftovers[3] = 0; /* No leftovers. */
             channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
             channel->rd_full = 0;
 
             spin_unlock_irqrestore(&channel->rd_spinlock, flags);
 
             iowrite32((channel->chan_num << 1) |
                   (4 << 24),   /* Opcode 4, open channel */
                   channel->endpoint->registers +
                   fpga_buf_ctrl_reg);
         }
 
         channel->rd_ref_count++;
     }
 
 unlock:
     if (filp->f_mode & FMODE_WRITE)
         mutex_unlock(&channel->rd_mutex);
 unlock_wr:
     if (filp->f_mode & FMODE_READ)
         mutex_unlock(&channel->wr_mutex);
 
     if (!rc && (!channel->seekable))
         return nonseekable_open(inode, filp);
 
     return rc;
 }
 
 static int xillybus_release(struct inode *inode, struct file *filp)
 {
     unsigned long flags;
     struct xilly_channel *channel = filp->private_data;
 
     int buf_idx;
     int eof;
 
     if (channel->endpoint->fatal_error)
         return -EIO;
 
     if (filp->f_mode & FMODE_WRITE) {
         mutex_lock(&channel->rd_mutex);
 
         channel->rd_ref_count--;
 
         if (channel->rd_ref_count == 0) {
             /*
              * We rely on the kernel calling flush()
              * before we get here.
              */
 
             iowrite32((channel->chan_num << 1) | /* Channel ID */
                   (5 << 24),  /* Opcode 5, close channel */
                   channel->endpoint->registers +
                   fpga_buf_ctrl_reg);
         }
         mutex_unlock(&channel->rd_mutex);
     }
 
     if (filp->f_mode & FMODE_READ) {
         mutex_lock(&channel->wr_mutex);
 
         channel->wr_ref_count--;
 
         if (channel->wr_ref_count == 0) {
             iowrite32(1 | (channel->chan_num << 1) |
                   (5 << 24),  /* Opcode 5, close channel */
                   channel->endpoint->registers +
                   fpga_buf_ctrl_reg);
 
             /*
              * This is crazily cautious: We make sure that not
              * only that we got an EOF (be it because we closed
              * the channel or because of a user's EOF), but verify
              * that it's one beyond the last buffer arrived, so
              * we have no leftover buffers pending before wrapping
              * up (which can only happen in asynchronous channels,
              * BTW)
              */
 
             while (1) {
                 spin_lock_irqsave(&channel->wr_spinlock,
                           flags);
                 buf_idx = channel->wr_fpga_buf_idx;
                 eof = channel->wr_eof;
                 channel->wr_sleepy = 1;
                 spin_unlock_irqrestore(&channel->wr_spinlock,
                                flags);
 
                 /*
                  * Check if eof points at the buffer after
                  * the last one the FPGA submitted. Note that
                  * no EOF is marked by negative eof.
                  */
 
                 buf_idx++;
                 if (buf_idx == channel->num_wr_buffers)
                     buf_idx = 0;
 
                 if (buf_idx == eof)
                     break;
 
                 /*
                  * Steal extra 100 ms if awaken by interrupt.
                  * This is a simple workaround for an
                  * interrupt pending when entering, which would
                  * otherwise result in declaring the hardware
                  * non-responsive.
                  */
 
                 if (wait_event_interruptible(
                         channel->wr_wait,
                         (!channel->wr_sleepy)))
                     msleep(100);
 
                 if (channel->wr_sleepy) {
                     mutex_unlock(&channel->wr_mutex);
                     dev_warn(channel->endpoint->dev,
                          "Hardware failed to respond to close command, therefore left in messy state.\n");
                     return -EINTR;
                 }
             }
         }
 
         mutex_unlock(&channel->wr_mutex);
     }
 
     return 0;
 }
 
 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
 {
     struct xilly_channel *channel = filp->private_data;
     loff_t pos = filp->f_pos;
     int rc = 0;
 
     /*
      * Take both mutexes not allowing interrupts, since it seems like
      * common applications don't expect an -EINTR here. Besides, multiple
      * access to a single file descriptor on seekable devices is a mess
      * anyhow.
      */
 
     if (channel->endpoint->fatal_error)
         return -EIO;
 
     mutex_lock(&channel->wr_mutex);
     mutex_lock(&channel->rd_mutex);
 
     switch (whence) {
     case SEEK_SET:
         pos = offset;
         break;
     case SEEK_CUR:
         pos += offset;
         break;
     case SEEK_END:
         pos = offset; /* Going to the end => to the beginning */
         break;
     default:
         rc = -EINVAL;
         goto end;
     }
 
     /* In any case, we must finish on an element boundary */
     if (pos & ((1 << channel->log2_element_size) - 1)) {
         rc = -EINVAL;
         goto end;
     }
 
     mutex_lock(&channel->endpoint->register_mutex);
 
     iowrite32(pos >> channel->log2_element_size,
           channel->endpoint->registers + fpga_buf_offset_reg);
 
     iowrite32((channel->chan_num << 1) |
           (6 << 24),  /* Opcode 6, set address */
           channel->endpoint->registers + fpga_buf_ctrl_reg);
 
     mutex_unlock(&channel->endpoint->register_mutex);
 
 end:
     mutex_unlock(&channel->rd_mutex);
     mutex_unlock(&channel->wr_mutex);
 
     if (rc) /* Return error after releasing mutexes */
         return rc;
 
     filp->f_pos = pos;
 
     /*
      * Since seekable devices are allowed only when the channel is
      * synchronous, we assume that there is no data pending in either
      * direction (which holds true as long as no concurrent access on the
      * file descriptor takes place).
      * The only thing we may need to throw away is leftovers from partial
      * write() flush.
      */
 
     channel->rd_leftovers[3] = 0;
 
     return pos;
 }
 
 static __poll_t xillybus_poll(struct file *filp, poll_table *wait)
 {
     struct xilly_channel *channel = filp->private_data;
     __poll_t mask = 0;
     unsigned long flags;
 
     poll_wait(filp, &channel->endpoint->ep_wait, wait);
 
     /*
      * poll() won't play ball regarding read() channels which
      * aren't asynchronous and support the nonempty message. Allowing
      * that will create situations where data has been delivered at
      * the FPGA, and users expecting select() to wake up, which it may
      * not.
      */
 
     if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
         poll_wait(filp, &channel->wr_wait, wait);
         poll_wait(filp, &channel->wr_ready_wait, wait);
 
         spin_lock_irqsave(&channel->wr_spinlock, flags);
         if (!channel->wr_empty || channel->wr_ready)
             mask |= EPOLLIN | EPOLLRDNORM;
 
         if (channel->wr_hangup)
             /*
              * Not EPOLLHUP, because its behavior is in the
              * mist, and EPOLLIN does what we want: Wake up
              * the read file descriptor so it sees EOF.
              */
             mask |=  EPOLLIN | EPOLLRDNORM;
         spin_unlock_irqrestore(&channel->wr_spinlock, flags);
     }
 
     /*
      * If partial data write is disallowed on a write() channel,
      * it's pointless to ever signal OK to write, because is could
      * block despite some space being available.
      */
 
     if (channel->rd_allow_partial) {
         poll_wait(filp, &channel->rd_wait, wait);
 
         spin_lock_irqsave(&channel->rd_spinlock, flags);
         if (!channel->rd_full)
             mask |= EPOLLOUT | EPOLLWRNORM;
         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
     }
 
     if (channel->endpoint->fatal_error)
         mask |= EPOLLERR;
 
     return mask;
 }
 
 static const struct file_operations xillybus_fops = {
     .owner      = THIS_MODULE,
     .read       = xillybus_read,
     .write      = xillybus_write,
     .open       = xillybus_open,
     .flush      = xillybus_flush,
     .release    = xillybus_release,
     .llseek     = xillybus_llseek,
     .poll       = xillybus_poll,
 };
 
 struct xilly_endpoint *xillybus_init_endpoint(struct device *dev)
 {
     struct xilly_endpoint *endpoint;
 
     endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);
     if (!endpoint)
         return NULL;
 
     endpoint->dev = dev;
     endpoint->msg_counter = 0x0b;
     endpoint->failed_messages = 0;
     endpoint->fatal_error = 0;
 
     init_waitqueue_head(&endpoint->ep_wait);
     mutex_init(&endpoint->register_mutex);
 
     return endpoint;
 }
 EXPORT_SYMBOL(xillybus_init_endpoint);
 
 static int xilly_quiesce(struct xilly_endpoint *endpoint)
 {
     long t;
 
     endpoint->idtlen = -1;
 
     iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
           endpoint->registers + fpga_dma_control_reg);
 
     t = wait_event_interruptible_timeout(endpoint->ep_wait,
                          (endpoint->idtlen >= 0),
                          XILLY_TIMEOUT);
     if (t <= 0) {
         dev_err(endpoint->dev,
             "Failed to quiesce the device on exit.\n");
         return -ENODEV;
     }
     return 0;
 }
 
 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
 {
     int rc;
     long t;
 
     void *bootstrap_resources;
     int idtbuffersize = (1 << PAGE_SHIFT);
     struct device *dev = endpoint->dev;
 
     /*
      * The bogus IDT is used during bootstrap for allocating the initial
      * message buffer, and then the message buffer and space for the IDT
      * itself. The initial message buffer is of a single page's size, but
      * it's soon replaced with a more modest one (and memory is freed).
      */
 
     unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
                        3, 192, PAGE_SHIFT, 0 };
     struct xilly_idt_handle idt_handle;
 
     /*
      * Writing the value 0x00000001 to Endianness register signals which
      * endianness this processor is using, so the FPGA can swap words as
      * necessary.
      */
 
     iowrite32(1, endpoint->registers + fpga_endian_reg);
 
     /* Bootstrap phase I: Allocate temporary message buffer */
 
     bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
     if (!bootstrap_resources)
         return -ENOMEM;
 
     endpoint->num_channels = 0;
 
     rc = xilly_setupchannels(endpoint, bogus_idt, 1);
     if (rc)
         return rc;
 
     /* Clear the message subsystem (and counter in particular) */
     iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
 
     endpoint->idtlen = -1;
 
     /*
      * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
      * buffer size.
      */
     iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
           endpoint->registers + fpga_dma_control_reg);
 
     t = wait_event_interruptible_timeout(endpoint->ep_wait,
                          (endpoint->idtlen >= 0),
                          XILLY_TIMEOUT);
     if (t <= 0) {
         dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
         return -ENODEV;
     }
 
     /* Enable DMA */
     iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
           endpoint->registers + fpga_dma_control_reg);
 
     /* Bootstrap phase II: Allocate buffer for IDT and obtain it */
     while (endpoint->idtlen >= idtbuffersize) {
         idtbuffersize *= 2;
         bogus_idt[6]++;
     }
 
     endpoint->num_channels = 1;
 
     rc = xilly_setupchannels(endpoint, bogus_idt, 2);
     if (rc)
         goto failed_idt;
 
     rc = xilly_obtain_idt(endpoint);
     if (rc)
         goto failed_idt;
 
     rc = xilly_scan_idt(endpoint, &idt_handle);
     if (rc)
         goto failed_idt;
 
     devres_close_group(dev, bootstrap_resources);
 
     /* Bootstrap phase III: Allocate buffers according to IDT */
 
     rc = xilly_setupchannels(endpoint,
                  idt_handle.chandesc,
                  idt_handle.entries);
     if (rc)
         goto failed_idt;
 
     rc = xillybus_init_chrdev(dev, &xillybus_fops,
                   endpoint->owner, endpoint,
                   idt_handle.names,
                   idt_handle.names_len,
                   endpoint->num_channels,
                   xillyname, false);
 
     if (rc)
         goto failed_idt;
 
     devres_release_group(dev, bootstrap_resources);
 
     return 0;
 
 failed_idt:
     xilly_quiesce(endpoint);
     flush_workqueue(xillybus_wq);
 
     return rc;
 }
 EXPORT_SYMBOL(xillybus_endpoint_discovery);
 
 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
 {
     xillybus_cleanup_chrdev(endpoint, endpoint->dev);
 
     xilly_quiesce(endpoint);
 
     /*
      * Flushing is done upon endpoint release to prevent access to memory
      * just about to be released. This makes the quiesce complete.
      */
     flush_workqueue(xillybus_wq);
 }
 EXPORT_SYMBOL(xillybus_endpoint_remove);
 
 static int __init xillybus_init(void)
 {
     xillybus_wq = alloc_workqueue(xillyname, 0, 0);
     if (!xillybus_wq)
         return -ENOMEM;
 
     return 0;
 }
 
 static void __exit xillybus_exit(void)
 {
     /* flush_workqueue() was called for each endpoint released */
     destroy_workqueue(xillybus_wq);
 }
 
 module_init(xillybus_init);
 module_exit(xillybus_exit);

This page was automatically generated by the 2.1.0 LXR engine. The LXR team