OSCL-LXR linux-6.0.1/​drivers/​usb/​musb/​cppi_dma.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2005-2006 by Texas Instruments
  *
  * This file implements a DMA  interface using TI's CPPI DMA.
  * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
  * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci.
  */
 
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/usb.h>
 
 #include "musb_core.h"
 #include "musb_debug.h"
 #include "cppi_dma.h"
 #include "davinci.h"
 
 
 /* CPPI DMA status 7-mar-2006:
  *
  * - See musb_{host,gadget}.c for more info
  *
  * - Correct RX DMA generally forces the engine into irq-per-packet mode,
  *   which can easily saturate the CPU under non-mass-storage loads.
  *
  * NOTES 24-aug-2006 (2.6.18-rc4):
  *
  * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
  *   evidently after the 1 byte packet was received and acked, the queue
  *   of BDs got garbaged so it wouldn't empty the fifo.  (rxcsr 0x2003,
  *   and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
  *   004001ff 00000001 .. 8feff860)  Host was just getting NAKed on tx
  *   of its next (512 byte) packet.  IRQ issues?
  *
  * REVISIT:  the "transfer DMA" glue between CPPI and USB fifos will
  * evidently also directly update the RX and TX CSRs ... so audit all
  * host and peripheral side DMA code to avoid CSR access after DMA has
  * been started.
  */
 
 /* REVISIT now we can avoid preallocating these descriptors; or
  * more simply, switch to a global freelist not per-channel ones.
  * Note: at full speed, 64 descriptors == 4K bulk data.
  */
 #define NUM_TXCHAN_BD       64
 #define NUM_RXCHAN_BD       64
 
 static inline void cpu_drain_writebuffer(void)
 {
     wmb();
 #ifdef  CONFIG_CPU_ARM926T
     /* REVISIT this "should not be needed",
      * but lack of it sure seemed to hurt ...
      */
     asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n");
 #endif
 }
 
 static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c)
 {
     struct cppi_descriptor  *bd = c->freelist;
 
     if (bd)
         c->freelist = bd->next;
     return bd;
 }
 
 static inline void
 cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd)
 {
     if (!bd)
         return;
     bd->next = c->freelist;
     c->freelist = bd;
 }
 
 /*
  *  Start DMA controller
  *
  *  Initialize the DMA controller as necessary.
  */
 
 /* zero out entire rx state RAM entry for the channel */
 static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx)
 {
     musb_writel(&rx->rx_skipbytes, 0, 0);
     musb_writel(&rx->rx_head, 0, 0);
     musb_writel(&rx->rx_sop, 0, 0);
     musb_writel(&rx->rx_current, 0, 0);
     musb_writel(&rx->rx_buf_current, 0, 0);
     musb_writel(&rx->rx_len_len, 0, 0);
     musb_writel(&rx->rx_cnt_cnt, 0, 0);
 }
 
 /* zero out entire tx state RAM entry for the channel */
 static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr)
 {
     musb_writel(&tx->tx_head, 0, 0);
     musb_writel(&tx->tx_buf, 0, 0);
     musb_writel(&tx->tx_current, 0, 0);
     musb_writel(&tx->tx_buf_current, 0, 0);
     musb_writel(&tx->tx_info, 0, 0);
     musb_writel(&tx->tx_rem_len, 0, 0);
     /* musb_writel(&tx->tx_dummy, 0, 0); */
     musb_writel(&tx->tx_complete, 0, ptr);
 }
 
 static void cppi_pool_init(struct cppi *cppi, struct cppi_channel *c)
 {
     int j;
 
     /* initialize channel fields */
     c->head = NULL;
     c->tail = NULL;
     c->last_processed = NULL;
     c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
     c->controller = cppi;
     c->is_rndis = 0;
     c->freelist = NULL;
 
     /* build the BD Free list for the channel */
     for (j = 0; j < NUM_TXCHAN_BD + 1; j++) {
         struct cppi_descriptor  *bd;
         dma_addr_t      dma;
 
         bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma);
         bd->dma = dma;
         cppi_bd_free(c, bd);
     }
 }
 
 static int cppi_channel_abort(struct dma_channel *);
 
 static void cppi_pool_free(struct cppi_channel *c)
 {
     struct cppi     *cppi = c->controller;
     struct cppi_descriptor  *bd;
 
     (void) cppi_channel_abort(&c->channel);
     c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
     c->controller = NULL;
 
     /* free all its bds */
     bd = c->last_processed;
     do {
         if (bd)
             dma_pool_free(cppi->pool, bd, bd->dma);
         bd = cppi_bd_alloc(c);
     } while (bd);
     c->last_processed = NULL;
 }
 
 static void cppi_controller_start(struct cppi *controller)
 {
     void __iomem    *tibase;
     int     i;
 
     /* do whatever is necessary to start controller */
     for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
         controller->tx[i].transmit = true;
         controller->tx[i].index = i;
     }
     for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
         controller->rx[i].transmit = false;
         controller->rx[i].index = i;
     }
 
     /* setup BD list on a per channel basis */
     for (i = 0; i < ARRAY_SIZE(controller->tx); i++)
         cppi_pool_init(controller, controller->tx + i);
     for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
         cppi_pool_init(controller, controller->rx + i);
 
     tibase =  controller->tibase;
     INIT_LIST_HEAD(&controller->tx_complete);
 
     /* initialise tx/rx channel head pointers to zero */
     for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
         struct cppi_channel *tx_ch = controller->tx + i;
         struct cppi_tx_stateram __iomem *tx;
 
         INIT_LIST_HEAD(&tx_ch->tx_complete);
 
         tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i);
         tx_ch->state_ram = tx;
         cppi_reset_tx(tx, 0);
     }
     for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
         struct cppi_channel *rx_ch = controller->rx + i;
         struct cppi_rx_stateram __iomem *rx;
 
         INIT_LIST_HEAD(&rx_ch->tx_complete);
 
         rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i);
         rx_ch->state_ram = rx;
         cppi_reset_rx(rx);
     }
 
     /* enable individual cppi channels */
     musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG,
             DAVINCI_DMA_ALL_CHANNELS_ENABLE);
     musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG,
             DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 
     /* enable tx/rx CPPI control */
     musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
     musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
 
     /* disable RNDIS mode, also host rx RNDIS autorequest */
     musb_writel(tibase, DAVINCI_RNDIS_REG, 0);
     musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0);
 }
 
 /*
  *  Stop DMA controller
  *
  *  De-Init the DMA controller as necessary.
  */
 
 static void cppi_controller_stop(struct cppi *controller)
 {
     void __iomem        *tibase;
     int         i;
     struct musb     *musb;
 
     musb = controller->controller.musb;
 
     tibase = controller->tibase;
     /* DISABLE INDIVIDUAL CHANNEL Interrupts */
     musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG,
             DAVINCI_DMA_ALL_CHANNELS_ENABLE);
     musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG,
             DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 
     musb_dbg(musb, "Tearing down RX and TX Channels");
     for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
         /* FIXME restructure of txdma to use bds like rxdma */
         controller->tx[i].last_processed = NULL;
         cppi_pool_free(controller->tx + i);
     }
     for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
         cppi_pool_free(controller->rx + i);
 
     /* in Tx Case proper teardown is supported. We resort to disabling
      * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is
      * complete TX CPPI cannot be disabled.
      */
     /*disable tx/rx cppi */
     musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
     musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
 }
 
 /* While dma channel is allocated, we only want the core irqs active
  * for fault reports, otherwise we'd get irqs that we don't care about.
  * Except for TX irqs, where dma done != fifo empty and reusable ...
  *
  * NOTE: docs don't say either way, but irq masking **enables** irqs.
  *
  * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
  */
 static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum)
 {
     musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8));
 }
 
 static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum)
 {
     musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8));
 }
 
 
 /*
  * Allocate a CPPI Channel for DMA.  With CPPI, channels are bound to
  * each transfer direction of a non-control endpoint, so allocating
  * (and deallocating) is mostly a way to notice bad housekeeping on
  * the software side.  We assume the irqs are always active.
  */
 static struct dma_channel *
 cppi_channel_allocate(struct dma_controller *c,
         struct musb_hw_ep *ep, u8 transmit)
 {
     struct cppi     *controller;
     u8          index;
     struct cppi_channel *cppi_ch;
     void __iomem        *tibase;
     struct musb     *musb;
 
     controller = container_of(c, struct cppi, controller);
     tibase = controller->tibase;
     musb = c->musb;
 
     /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */
     index = ep->epnum - 1;
 
     /* return the corresponding CPPI Channel Handle, and
      * probably disable the non-CPPI irq until we need it.
      */
     if (transmit) {
         if (index >= ARRAY_SIZE(controller->tx)) {
             musb_dbg(musb, "no %cX%d CPPI channel", 'T', index);
             return NULL;
         }
         cppi_ch = controller->tx + index;
     } else {
         if (index >= ARRAY_SIZE(controller->rx)) {
             musb_dbg(musb, "no %cX%d CPPI channel", 'R', index);
             return NULL;
         }
         cppi_ch = controller->rx + index;
         core_rxirq_disable(tibase, ep->epnum);
     }
 
     /* REVISIT make this an error later once the same driver code works
      * with the other DMA engine too
      */
     if (cppi_ch->hw_ep)
         musb_dbg(musb, "re-allocating DMA%d %cX channel %p",
                 index, transmit ? 'T' : 'R', cppi_ch);
     cppi_ch->hw_ep = ep;
     cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
     cppi_ch->channel.max_len = 0x7fffffff;
 
     musb_dbg(musb, "Allocate CPPI%d %cX", index, transmit ? 'T' : 'R');
     return &cppi_ch->channel;
 }
 
 /* Release a CPPI Channel.  */
 static void cppi_channel_release(struct dma_channel *channel)
 {
     struct cppi_channel *c;
     void __iomem        *tibase;
 
     /* REVISIT:  for paranoia, check state and abort if needed... */
 
     c = container_of(channel, struct cppi_channel, channel);
     tibase = c->controller->tibase;
     if (!c->hw_ep)
         musb_dbg(c->controller->controller.musb,
             "releasing idle DMA channel %p", c);
     else if (!c->transmit)
         core_rxirq_enable(tibase, c->index + 1);
 
     /* for now, leave its cppi IRQ enabled (we won't trigger it) */
     c->hw_ep = NULL;
     channel->status = MUSB_DMA_STATUS_UNKNOWN;
 }
 
 /* Context: controller irqlocked */
 static void
 cppi_dump_rx(int level, struct cppi_channel *c, const char *tag)
 {
     void __iomem            *base = c->controller->mregs;
     struct cppi_rx_stateram __iomem *rx = c->state_ram;
 
     musb_ep_select(base, c->index + 1);
 
     musb_dbg(c->controller->controller.musb,
         "RX DMA%d%s: %d left, csr %04x, "
         "%08x H%08x S%08x C%08x, "
         "B%08x L%08x %08x .. %08x",
         c->index, tag,
         musb_readl(c->controller->tibase,
             DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index),
         musb_readw(c->hw_ep->regs, MUSB_RXCSR),
 
         musb_readl(&rx->rx_skipbytes, 0),
         musb_readl(&rx->rx_head, 0),
         musb_readl(&rx->rx_sop, 0),
         musb_readl(&rx->rx_current, 0),
 
         musb_readl(&rx->rx_buf_current, 0),
         musb_readl(&rx->rx_len_len, 0),
         musb_readl(&rx->rx_cnt_cnt, 0),
         musb_readl(&rx->rx_complete, 0)
         );
 }
 
 /* Context: controller irqlocked */
 static void
 cppi_dump_tx(int level, struct cppi_channel *c, const char *tag)
 {
     void __iomem            *base = c->controller->mregs;
     struct cppi_tx_stateram __iomem *tx = c->state_ram;
 
     musb_ep_select(base, c->index + 1);
 
     musb_dbg(c->controller->controller.musb,
         "TX DMA%d%s: csr %04x, "
         "H%08x S%08x C%08x %08x, "
         "F%08x L%08x .. %08x",
         c->index, tag,
         musb_readw(c->hw_ep->regs, MUSB_TXCSR),
 
         musb_readl(&tx->tx_head, 0),
         musb_readl(&tx->tx_buf, 0),
         musb_readl(&tx->tx_current, 0),
         musb_readl(&tx->tx_buf_current, 0),
 
         musb_readl(&tx->tx_info, 0),
         musb_readl(&tx->tx_rem_len, 0),
         /* dummy/unused word 6 */
         musb_readl(&tx->tx_complete, 0)
         );
 }
 
 /* Context: controller irqlocked */
 static inline void
 cppi_rndis_update(struct cppi_channel *c, int is_rx,
         void __iomem *tibase, int is_rndis)
 {
     /* we may need to change the rndis flag for this cppi channel */
     if (c->is_rndis != is_rndis) {
         u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG);
         u32 temp = 1 << (c->index);
 
         if (is_rx)
             temp <<= 16;
         if (is_rndis)
             value |= temp;
         else
             value &= ~temp;
         musb_writel(tibase, DAVINCI_RNDIS_REG, value);
         c->is_rndis = is_rndis;
     }
 }
 
 static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd)
 {
     pr_debug("RXBD/%s %08x: "
             "nxt %08x buf %08x off.blen %08x opt.plen %08x\n",
             tag, bd->dma,
             bd->hw_next, bd->hw_bufp, bd->hw_off_len,
             bd->hw_options);
 }
 
 static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
 {
     struct cppi_descriptor  *bd;
 
     cppi_dump_rx(level, rx, tag);
     if (rx->last_processed)
         cppi_dump_rxbd("last", rx->last_processed);
     for (bd = rx->head; bd; bd = bd->next)
         cppi_dump_rxbd("active", bd);
 }
 
 
 /* NOTE:  DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
  * so we won't ever use it (see "CPPI RX Woes" below).
  */
 static inline int cppi_autoreq_update(struct cppi_channel *rx,
         void __iomem *tibase, int onepacket, unsigned n_bds)
 {
     u32 val;
 
 #ifdef  RNDIS_RX_IS_USABLE
     u32 tmp;
     /* assert(is_host_active(musb)) */
 
     /* start from "AutoReq never" */
     tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
     val = tmp & ~((0x3) << (rx->index * 2));
 
     /* HCD arranged reqpkt for packet #1.  we arrange int
      * for all but the last one, maybe in two segments.
      */
     if (!onepacket) {
 #if 0
         /* use two segments, autoreq "all" then the last "never" */
         val |= ((0x3) << (rx->index * 2));
         n_bds--;
 #else
         /* one segment, autoreq "all-but-last" */
         val |= ((0x1) << (rx->index * 2));
 #endif
     }
 
     if (val != tmp) {
         int n = 100;
 
         /* make sure that autoreq is updated before continuing */
         musb_writel(tibase, DAVINCI_AUTOREQ_REG, val);
         do {
             tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
             if (tmp == val)
                 break;
             cpu_relax();
         } while (n-- > 0);
     }
 #endif
 
     /* REQPKT is turned off after each segment */
     if (n_bds && rx->channel.actual_len) {
         void __iomem    *regs = rx->hw_ep->regs;
 
         val = musb_readw(regs, MUSB_RXCSR);
         if (!(val & MUSB_RXCSR_H_REQPKT)) {
             val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
             musb_writew(regs, MUSB_RXCSR, val);
             /* flush writebuffer */
             val = musb_readw(regs, MUSB_RXCSR);
         }
     }
     return n_bds;
 }
 
 
 /* Buffer enqueuing Logic:
  *
  *  - RX builds new queues each time, to help handle routine "early
  *    termination" cases (faults, including errors and short reads)
  *    more correctly.
  *
  *  - for now, TX reuses the same queue of BDs every time
  *
  * REVISIT long term, we want a normal dynamic model.
  * ... the goal will be to append to the
  * existing queue, processing completed "dma buffers" (segments) on the fly.
  *
  * Otherwise we force an IRQ latency between requests, which slows us a lot
  * (especially in "transparent" dma).  Unfortunately that model seems to be
  * inherent in the DMA model from the Mentor code, except in the rare case
  * of transfers big enough (~128+ KB) that we could append "middle" segments
  * in the TX paths.  (RX can't do this, see below.)
  *
  * That's true even in the CPPI- friendly iso case, where most urbs have
  * several small segments provided in a group and where the "packet at a time"
  * "transparent" DMA model is always correct, even on the RX side.
  */
 
 /*
  * CPPI TX:
  * ========
  * TX is a lot more reasonable than RX; it doesn't need to run in
  * irq-per-packet mode very often.  RNDIS mode seems to behave too
  * (except how it handles the exactly-N-packets case).  Building a
  * txdma queue with multiple requests (urb or usb_request) looks
  * like it would work ... but fault handling would need much testing.
  *
  * The main issue with TX mode RNDIS relates to transfer lengths that
  * are an exact multiple of the packet length.  It appears that there's
  * a hiccup in that case (maybe the DMA completes before the ZLP gets
  * written?) boiling down to not being able to rely on CPPI writing any
  * terminating zero length packet before the next transfer is written.
  * So that's punted to PIO; better yet, gadget drivers can avoid it.
  *
  * Plus, there's allegedly an undocumented constraint that rndis transfer
  * length be a multiple of 64 bytes ... but the chip doesn't act that
  * way, and we really don't _want_ that behavior anyway.
  *
  * On TX, "transparent" mode works ... although experiments have shown
  * problems trying to use the SOP/EOP bits in different USB packets.
  *
  * REVISIT try to handle terminating zero length packets using CPPI
  * instead of doing it by PIO after an IRQ.  (Meanwhile, make Ethernet
  * links avoid that issue by forcing them to avoid zlps.)
  */
 static void
 cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx)
 {
     unsigned        maxpacket = tx->maxpacket;
     dma_addr_t      addr = tx->buf_dma + tx->offset;
     size_t          length = tx->buf_len - tx->offset;
     struct cppi_descriptor  *bd;
     unsigned        n_bds;
     unsigned        i;
     struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram;
     int         rndis;
 
     /* TX can use the CPPI "rndis" mode, where we can probably fit this
      * transfer in one BD and one IRQ.  The only time we would NOT want
      * to use it is when hardware constraints prevent it, or if we'd
      * trigger the "send a ZLP?" confusion.
      */
     rndis = (maxpacket & 0x3f) == 0
         && length > maxpacket
         && length < 0xffff
         && (length % maxpacket) != 0;
 
     if (rndis) {
         maxpacket = length;
         n_bds = 1;
     } else {
         if (length)
             n_bds = DIV_ROUND_UP(length, maxpacket);
         else
             n_bds = 1;
         n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD);
         length = min(n_bds * maxpacket, length);
     }
 
     musb_dbg(musb, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u",
             tx->index,
             maxpacket,
             rndis ? "rndis" : "transparent",
             n_bds,
             (unsigned long long)addr, length);
 
     cppi_rndis_update(tx, 0, musb->ctrl_base, rndis);
 
     /* assuming here that channel_program is called during
      * transfer initiation ... current code maintains state
      * for one outstanding request only (no queues, not even
      * the implicit ones of an iso urb).
      */
 
     bd = tx->freelist;
     tx->head = bd;
     tx->last_processed = NULL;
 
     /* FIXME use BD pool like RX side does, and just queue
      * the minimum number for this request.
      */
 
     /* Prepare queue of BDs first, then hand it to hardware.
      * All BDs except maybe the last should be of full packet
      * size; for RNDIS there _is_ only that last packet.
      */
     for (i = 0; i < n_bds; ) {
         if (++i < n_bds && bd->next)
             bd->hw_next = bd->next->dma;
         else
             bd->hw_next = 0;
 
         bd->hw_bufp = tx->buf_dma + tx->offset;
 
         /* FIXME set EOP only on the last packet,
          * SOP only on the first ... avoid IRQs
          */
         if ((tx->offset + maxpacket) <= tx->buf_len) {
             tx->offset += maxpacket;
             bd->hw_off_len = maxpacket;
             bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
                 | CPPI_OWN_SET | maxpacket;
         } else {
             /* only this one may be a partial USB Packet */
             u32     partial_len;
 
             partial_len = tx->buf_len - tx->offset;
             tx->offset = tx->buf_len;
             bd->hw_off_len = partial_len;
 
             bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
                 | CPPI_OWN_SET | partial_len;
             if (partial_len == 0)
                 bd->hw_options |= CPPI_ZERO_SET;
         }
 
         musb_dbg(musb, "TXBD %p: nxt %08x buf %08x len %04x opt %08x",
                 bd, bd->hw_next, bd->hw_bufp,
                 bd->hw_off_len, bd->hw_options);
 
         /* update the last BD enqueued to the list */
         tx->tail = bd;
         bd = bd->next;
     }
 
     /* BDs live in DMA-coherent memory, but writes might be pending */
     cpu_drain_writebuffer();
 
     /* Write to the HeadPtr in state RAM to trigger */
     musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma);
 
     cppi_dump_tx(5, tx, "/S");
 }
 
 /*
  * CPPI RX Woes:
  * =============
  * Consider a 1KB bulk RX buffer in two scenarios:  (a) it's fed two 300 byte
  * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
  * (Full speed transfers have similar scenarios.)
  *
  * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
  * and the next packet goes into a buffer that's queued later; while (b) fills
  * the buffer with 1024 bytes.  How to do that with CPPI?
  *
  * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
  *   (b) loses **BADLY** because nothing (!) happens when that second packet
  *   fills the buffer, much less when a third one arrives.  (Which makes this
  *   not a "true" RNDIS mode.  In the RNDIS protocol short-packet termination
  *   is optional, and it's fine if peripherals -- not hosts! -- pad messages
  *   out to end-of-buffer.  Standard PCI host controller DMA descriptors
  *   implement that mode by default ... which is no accident.)
  *
  * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
  *   converse problems:  (b) is handled right, but (a) loses badly.  CPPI RX
  *   ignores SOP/EOP markings and processes both of those BDs; so both packets
  *   are loaded into the buffer (with a 212 byte gap between them), and the next
  *   buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP
  *   are intended as outputs for RX queues, not inputs...)
  *
  * - A variant of "transparent" mode -- one BD at a time -- is the only way to
  *   reliably make both cases work, with software handling both cases correctly
  *   and at the significant penalty of needing an IRQ per packet.  (The lack of
  *   I/O overlap can be slightly ameliorated by enabling double buffering.)
  *
  * So how to get rid of IRQ-per-packet?  The transparent multi-BD case could
  * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK
  * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors
  * with guaranteed driver level fault recovery and scrubbing out what's left
  * of that garbaged datastream.
  *
  * But there seems to be no way to identify the cases where CPPI RNDIS mode
  * is appropriate -- which do NOT include RNDIS host drivers, but do include
  * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
  * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
  * that applies best on the peripheral side (and which could fail rudely).
  *
  * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
  * cases other than mass storage class.  Otherwise we're correct but slow,
  * since CPPI penalizes our need for a "true RNDIS" default mode.
  */
 
 
 /* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY
  *
  * IFF
  *  (a) peripheral mode ... since rndis peripherals could pad their
  *  writes to hosts, causing i/o failure; or we'd have to cope with
  *  a largely unknowable variety of host side protocol variants
  *  (b) and short reads are NOT errors ... since full reads would
  *  cause those same i/o failures
  *  (c) and read length is
  *  - less than 64KB (max per cppi descriptor)
  *  - not a multiple of 4096 (g_zero default, full reads typical)
  *  - N (>1) packets long, ditto (full reads not EXPECTED)
  * THEN
  *   try rx rndis mode
  *
  * Cost of heuristic failing:  RXDMA wedges at the end of transfers that
  * fill out the whole buffer.  Buggy host side usb network drivers could
  * trigger that, but "in the field" such bugs seem to be all but unknown.
  *
  * So this module parameter lets the heuristic be disabled.  When using
  * gadgetfs, the heuristic will probably need to be disabled.
  */
 static bool cppi_rx_rndis = 1;
 
 module_param(cppi_rx_rndis, bool, 0);
 MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
 
 
 /**
  * cppi_next_rx_segment - dma read for the next chunk of a buffer
  * @musb: the controller
  * @rx: dma channel
  * @onepacket: true unless caller treats short reads as errors, and
  *  performs fault recovery above usbcore.
  * Context: controller irqlocked
  *
  * See above notes about why we can't use multi-BD RX queues except in
  * rare cases (mass storage class), and can never use the hardware "rndis"
  * mode (since it's not a "true" RNDIS mode) with complete safety..
  *
  * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in
  * code to recover from corrupted datastreams after each short transfer.
  */
 static void
 cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket)
 {
     unsigned        maxpacket = rx->maxpacket;
     dma_addr_t      addr = rx->buf_dma + rx->offset;
     size_t          length = rx->buf_len - rx->offset;
     struct cppi_descriptor  *bd, *tail;
     unsigned        n_bds;
     unsigned        i;
     void __iomem        *tibase = musb->ctrl_base;
     int         is_rndis = 0;
     struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram;
     struct cppi_descriptor  *d;
 
     if (onepacket) {
         /* almost every USB driver, host or peripheral side */
         n_bds = 1;
 
         /* maybe apply the heuristic above */
         if (cppi_rx_rndis
                 && is_peripheral_active(musb)
                 && length > maxpacket
                 && (length & ~0xffff) == 0
                 && (length & 0x0fff) != 0
                 && (length & (maxpacket - 1)) == 0) {
             maxpacket = length;
             is_rndis = 1;
         }
     } else {
         /* virtually nothing except mass storage class */
         if (length > 0xffff) {
             n_bds = 0xffff / maxpacket;
             length = n_bds * maxpacket;
         } else {
             n_bds = DIV_ROUND_UP(length, maxpacket);
         }
         if (n_bds == 1)
             onepacket = 1;
         else
             n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD);
     }
 
     /* In host mode, autorequest logic can generate some IN tokens; it's
      * tricky since we can't leave REQPKT set in RXCSR after the transfer
      * finishes. So:  multipacket transfers involve two or more segments.
      * And always at least two IRQs ... RNDIS mode is not an option.
      */
     if (is_host_active(musb))
         n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds);
 
     cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis);
 
     length = min(n_bds * maxpacket, length);
 
     musb_dbg(musb, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
             "dma 0x%llx len %u %u/%u",
             rx->index, maxpacket,
             onepacket
                 ? (is_rndis ? "rndis" : "onepacket")
                 : "multipacket",
             n_bds,
             musb_readl(tibase,
                 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
                     & 0xffff,
             (unsigned long long)addr, length,
             rx->channel.actual_len, rx->buf_len);
 
     /* only queue one segment at a time, since the hardware prevents
      * correct queue shutdown after unexpected short packets
      */
     bd = cppi_bd_alloc(rx);
     rx->head = bd;
 
     /* Build BDs for all packets in this segment */
     for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) {
         u32 bd_len;
 
         if (i) {
             bd = cppi_bd_alloc(rx);
             if (!bd)
                 break;
             tail->next = bd;
             tail->hw_next = bd->dma;
         }
         bd->hw_next = 0;
 
         /* all but the last packet will be maxpacket size */
         if (maxpacket < length)
             bd_len = maxpacket;
         else
             bd_len = length;
 
         bd->hw_bufp = addr;
         addr += bd_len;
         rx->offset += bd_len;
 
         bd->hw_off_len = (0 /*offset*/ << 16) + bd_len;
         bd->buflen = bd_len;
 
         bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0);
         length -= bd_len;
     }
 
     /* we always expect at least one reusable BD! */
     if (!tail) {
         WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds);
         return;
     } else if (i < n_bds)
         WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds);
 
     tail->next = NULL;
     tail->hw_next = 0;
 
     bd = rx->head;
     rx->tail = tail;
 
     /* short reads and other faults should terminate this entire
      * dma segment.  we want one "dma packet" per dma segment, not
      * one per USB packet, terminating the whole queue at once...
      * NOTE that current hardware seems to ignore SOP and EOP.
      */
     bd->hw_options |= CPPI_SOP_SET;
     tail->hw_options |= CPPI_EOP_SET;
 
     for (d = rx->head; d; d = d->next)
         cppi_dump_rxbd("S", d);
 
     /* in case the preceding transfer left some state... */
     tail = rx->last_processed;
     if (tail) {
         tail->next = bd;
         tail->hw_next = bd->dma;
     }
 
     core_rxirq_enable(tibase, rx->index + 1);
 
     /* BDs live in DMA-coherent memory, but writes might be pending */
     cpu_drain_writebuffer();
 
     /* REVISIT specs say to write this AFTER the BUFCNT register
      * below ... but that loses badly.
      */
     musb_writel(&rx_ram->rx_head, 0, bd->dma);
 
     /* bufferCount must be at least 3, and zeroes on completion
      * unless it underflows below zero, or stops at two, or keeps
      * growing ... grr.
      */
     i = musb_readl(tibase,
             DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
             & 0xffff;
 
     if (!i)
         musb_writel(tibase,
             DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
             n_bds + 2);
     else if (n_bds > (i - 3))
         musb_writel(tibase,
             DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
             n_bds - (i - 3));
 
     i = musb_readl(tibase,
             DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
             & 0xffff;
     if (i < (2 + n_bds)) {
         musb_dbg(musb, "bufcnt%d underrun - %d (for %d)",
                     rx->index, i, n_bds);
         musb_writel(tibase,
             DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
             n_bds + 2);
     }
 
     cppi_dump_rx(4, rx, "/S");
 }
 
 /**
  * cppi_channel_program - program channel for data transfer
  * @ch: the channel
  * @maxpacket: max packet size
  * @mode: For RX, 1 unless the usb protocol driver promised to treat
  *  all short reads as errors and kick in high level fault recovery.
  *  For TX, ignored because of RNDIS mode races/glitches.
  * @dma_addr: dma address of buffer
  * @len: length of buffer
  * Context: controller irqlocked
  */
 static int cppi_channel_program(struct dma_channel *ch,
         u16 maxpacket, u8 mode,
         dma_addr_t dma_addr, u32 len)
 {
     struct cppi_channel *cppi_ch;
     struct cppi     *controller;
     struct musb     *musb;
 
     cppi_ch = container_of(ch, struct cppi_channel, channel);
     controller = cppi_ch->controller;
     musb = controller->controller.musb;
 
     switch (ch->status) {
     case MUSB_DMA_STATUS_BUS_ABORT:
     case MUSB_DMA_STATUS_CORE_ABORT:
         /* fault irq handler should have handled cleanup */
         WARNING("%cX DMA%d not cleaned up after abort!\n",
                 cppi_ch->transmit ? 'T' : 'R',
                 cppi_ch->index);
         /* WARN_ON(1); */
         break;
     case MUSB_DMA_STATUS_BUSY:
         WARNING("program active channel?  %cX DMA%d\n",
                 cppi_ch->transmit ? 'T' : 'R',
                 cppi_ch->index);
         /* WARN_ON(1); */
         break;
     case MUSB_DMA_STATUS_UNKNOWN:
         musb_dbg(musb, "%cX DMA%d not allocated!",
                 cppi_ch->transmit ? 'T' : 'R',
                 cppi_ch->index);
         fallthrough;
     case MUSB_DMA_STATUS_FREE:
         break;
     }
 
     ch->status = MUSB_DMA_STATUS_BUSY;
 
     /* set transfer parameters, then queue up its first segment */
     cppi_ch->buf_dma = dma_addr;
     cppi_ch->offset = 0;
     cppi_ch->maxpacket = maxpacket;
     cppi_ch->buf_len = len;
     cppi_ch->channel.actual_len = 0;
 
     /* TX channel? or RX? */
     if (cppi_ch->transmit)
         cppi_next_tx_segment(musb, cppi_ch);
     else
         cppi_next_rx_segment(musb, cppi_ch, mode);
 
     return true;
 }
 
 static bool cppi_rx_scan(struct cppi *cppi, unsigned ch)
 {
     struct cppi_channel     *rx = &cppi->rx[ch];
     struct cppi_rx_stateram __iomem *state = rx->state_ram;
     struct cppi_descriptor      *bd;
     struct cppi_descriptor      *last = rx->last_processed;
     bool                completed = false;
     bool                acked = false;
     int             i;
     dma_addr_t          safe2ack;
     void __iomem            *regs = rx->hw_ep->regs;
     struct musb         *musb = cppi->controller.musb;
 
     cppi_dump_rx(6, rx, "/K");
 
     bd = last ? last->next : rx->head;
     if (!bd)
         return false;
 
     /* run through all completed BDs */
     for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0);
             (safe2ack || completed) && bd && i < NUM_RXCHAN_BD;
             i++, bd = bd->next) {
         u16 len;
 
         /* catch latest BD writes from CPPI */
         rmb();
         if (!completed && (bd->hw_options & CPPI_OWN_SET))
             break;
 
         musb_dbg(musb, "C/RXBD %llx: nxt %08x buf %08x "
             "off.len %08x opt.len %08x (%d)",
             (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp,
             bd->hw_off_len, bd->hw_options,
             rx->channel.actual_len);
 
         /* actual packet received length */
         if ((bd->hw_options & CPPI_SOP_SET) && !completed)
             len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK;
         else
             len = 0;
 
         if (bd->hw_options & CPPI_EOQ_MASK)
             completed = true;
 
         if (!completed && len < bd->buflen) {
             /* NOTE:  when we get a short packet, RXCSR_H_REQPKT
              * must have been cleared, and no more DMA packets may
              * active be in the queue... TI docs didn't say, but
              * CPPI ignores those BDs even though OWN is still set.
              */
             completed = true;
             musb_dbg(musb, "rx short %d/%d (%d)",
                     len, bd->buflen,
                     rx->channel.actual_len);
         }
 
         /* If we got here, we expect to ack at least one BD; meanwhile
          * CPPI may completing other BDs while we scan this list...
          *
          * RACE: we can notice OWN cleared before CPPI raises the
          * matching irq by writing that BD as the completion pointer.
          * In such cases, stop scanning and wait for the irq, avoiding
          * lost acks and states where BD ownership is unclear.
          */
         if (bd->dma == safe2ack) {
             musb_writel(&state->rx_complete, 0, safe2ack);
             safe2ack = musb_readl(&state->rx_complete, 0);
             acked = true;
             if (bd->dma == safe2ack)
                 safe2ack = 0;
         }
 
         rx->channel.actual_len += len;
 
         cppi_bd_free(rx, last);
         last = bd;
 
         /* stop scanning on end-of-segment */
         if (bd->hw_next == 0)
             completed = true;
     }
     rx->last_processed = last;
 
     /* dma abort, lost ack, or ... */
     if (!acked && last) {
         int csr;
 
         if (safe2ack == 0 || safe2ack == rx->last_processed->dma)
             musb_writel(&state->rx_complete, 0, safe2ack);
         if (safe2ack == 0) {
             cppi_bd_free(rx, last);
             rx->last_processed = NULL;
 
             /* if we land here on the host side, H_REQPKT will
              * be clear and we need to restart the queue...
              */
             WARN_ON(rx->head);
         }
         musb_ep_select(cppi->mregs, rx->index + 1);
         csr = musb_readw(regs, MUSB_RXCSR);
         if (csr & MUSB_RXCSR_DMAENAB) {
             musb_dbg(musb, "list%d %p/%p, last %llx%s, csr %04x",
                 rx->index,
                 rx->head, rx->tail,
                 rx->last_processed
                     ? (unsigned long long)
                         rx->last_processed->dma
                     : 0,
                 completed ? ", completed" : "",
                 csr);
             cppi_dump_rxq(4, "/what?", rx);
         }
     }
     if (!completed) {
         int csr;
 
         rx->head = bd;
 
         /* REVISIT seems like "autoreq all but EOP" doesn't...
          * setting it here "should" be racey, but seems to work
          */
         csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
         if (is_host_active(cppi->controller.musb)
                 && bd
                 && !(csr & MUSB_RXCSR_H_REQPKT)) {
             csr |= MUSB_RXCSR_H_REQPKT;
             musb_writew(regs, MUSB_RXCSR,
                     MUSB_RXCSR_H_WZC_BITS | csr);
             csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
         }
     } else {
         rx->head = NULL;
         rx->tail = NULL;
     }
 
     cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned");
     return completed;
 }
 
 irqreturn_t cppi_interrupt(int irq, void *dev_id)
 {
     struct musb     *musb = dev_id;
     struct cppi     *cppi;
     void __iomem        *tibase;
     struct musb_hw_ep   *hw_ep = NULL;
     u32         rx, tx;
     int         i, index;
     unsigned long       flags;
 
     cppi = container_of(musb->dma_controller, struct cppi, controller);
     if (cppi->irq)
         spin_lock_irqsave(&musb->lock, flags);
 
     tibase = musb->ctrl_base;
 
     tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG);
     rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG);
 
     if (!tx && !rx) {
         if (cppi->irq)
             spin_unlock_irqrestore(&musb->lock, flags);
         return IRQ_NONE;
     }
 
     musb_dbg(musb, "CPPI IRQ Tx%x Rx%x", tx, rx);
 
     /* process TX channels */
     for (index = 0; tx; tx = tx >> 1, index++) {
         struct cppi_channel     *tx_ch;
         struct cppi_tx_stateram __iomem *tx_ram;
         bool                completed = false;
         struct cppi_descriptor      *bd;
 
         if (!(tx & 1))
             continue;
 
         tx_ch = cppi->tx + index;
         tx_ram = tx_ch->state_ram;
 
         /* FIXME  need a cppi_tx_scan() routine, which
          * can also be called from abort code
          */
 
         cppi_dump_tx(5, tx_ch, "/E");
 
         bd = tx_ch->head;
 
         /*
          * If Head is null then this could mean that a abort interrupt
          * that needs to be acknowledged.
          */
         if (NULL == bd) {
             musb_dbg(musb, "null BD");
             musb_writel(&tx_ram->tx_complete, 0, 0);
             continue;
         }
 
         /* run through all completed BDs */
         for (i = 0; !completed && bd && i < NUM_TXCHAN_BD;
                 i++, bd = bd->next) {
             u16 len;
 
             /* catch latest BD writes from CPPI */
             rmb();
             if (bd->hw_options & CPPI_OWN_SET)
                 break;
 
             musb_dbg(musb, "C/TXBD %p n %x b %x off %x opt %x",
                     bd, bd->hw_next, bd->hw_bufp,
                     bd->hw_off_len, bd->hw_options);
 
             len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK;
             tx_ch->channel.actual_len += len;
 
             tx_ch->last_processed = bd;
 
             /* write completion register to acknowledge
              * processing of completed BDs, and possibly
              * release the IRQ; EOQ might not be set ...
              *
              * REVISIT use the same ack strategy as rx
              *
              * REVISIT have observed bit 18 set; huh??
              */
             /* if ((bd->hw_options & CPPI_EOQ_MASK)) */
                 musb_writel(&tx_ram->tx_complete, 0, bd->dma);
 
             /* stop scanning on end-of-segment */
             if (bd->hw_next == 0)
                 completed = true;
         }
 
         /* on end of segment, maybe go to next one */
         if (completed) {
             /* cppi_dump_tx(4, tx_ch, "/complete"); */
 
             /* transfer more, or report completion */
             if (tx_ch->offset >= tx_ch->buf_len) {
                 tx_ch->head = NULL;
                 tx_ch->tail = NULL;
                 tx_ch->channel.status = MUSB_DMA_STATUS_FREE;
 
                 hw_ep = tx_ch->hw_ep;
 
                 musb_dma_completion(musb, index + 1, 1);
 
             } else {
                 /* Bigger transfer than we could fit in
                  * that first batch of descriptors...
                  */
                 cppi_next_tx_segment(musb, tx_ch);
             }
         } else
             tx_ch->head = bd;
     }
 
     /* Start processing the RX block */
     for (index = 0; rx; rx = rx >> 1, index++) {
 
         if (rx & 1) {
             struct cppi_channel     *rx_ch;
 
             rx_ch = cppi->rx + index;
 
             /* let incomplete dma segments finish */
             if (!cppi_rx_scan(cppi, index))
                 continue;
 
             /* start another dma segment if needed */
             if (rx_ch->channel.actual_len != rx_ch->buf_len
                     && rx_ch->channel.actual_len
                         == rx_ch->offset) {
                 cppi_next_rx_segment(musb, rx_ch, 1);
                 continue;
             }
 
             /* all segments completed! */
             rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
 
             hw_ep = rx_ch->hw_ep;
 
             core_rxirq_disable(tibase, index + 1);
             musb_dma_completion(musb, index + 1, 0);
         }
     }
 
     /* write to CPPI EOI register to re-enable interrupts */
     musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0);
 
     if (cppi->irq)
         spin_unlock_irqrestore(&musb->lock, flags);
 
     return IRQ_HANDLED;
 }
 EXPORT_SYMBOL_GPL(cppi_interrupt);
 
 /* Instantiate a software object representing a DMA controller. */
 struct dma_controller *
 cppi_dma_controller_create(struct musb *musb, void __iomem *mregs)
 {
     struct cppi     *controller;
     struct device       *dev = musb->controller;
     struct platform_device  *pdev = to_platform_device(dev);
     int         irq = platform_get_irq_byname(pdev, "dma");
 
     controller = kzalloc(sizeof *controller, GFP_KERNEL);
     if (!controller)
         return NULL;
 
     controller->mregs = mregs;
     controller->tibase = mregs - DAVINCI_BASE_OFFSET;
 
     controller->controller.musb = musb;
     controller->controller.channel_alloc = cppi_channel_allocate;
     controller->controller.channel_release = cppi_channel_release;
     controller->controller.channel_program = cppi_channel_program;
     controller->controller.channel_abort = cppi_channel_abort;
 
     /* NOTE: allocating from on-chip SRAM would give the least
      * contention for memory access, if that ever matters here.
      */
 
     /* setup BufferPool */
     controller->pool = dma_pool_create("cppi",
             controller->controller.musb->controller,
             sizeof(struct cppi_descriptor),
             CPPI_DESCRIPTOR_ALIGN, 0);
     if (!controller->pool) {
         kfree(controller);
         return NULL;
     }
 
     if (irq > 0) {
         if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) {
             dev_err(dev, "request_irq %d failed!\n", irq);
             musb_dma_controller_destroy(&controller->controller);
             return NULL;
         }
         controller->irq = irq;
     }
 
     cppi_controller_start(controller);
     return &controller->controller;
 }
 EXPORT_SYMBOL_GPL(cppi_dma_controller_create);
 
 /*
  *  Destroy a previously-instantiated DMA controller.
  */
 void cppi_dma_controller_destroy(struct dma_controller *c)
 {
     struct cppi *cppi;
 
     cppi = container_of(c, struct cppi, controller);
 
     cppi_controller_stop(cppi);
 
     if (cppi->irq)
         free_irq(cppi->irq, cppi->controller.musb);
 
     /* assert:  caller stopped the controller first */
     dma_pool_destroy(cppi->pool);
 
     kfree(cppi);
 }
 EXPORT_SYMBOL_GPL(cppi_dma_controller_destroy);
 
 /*
  * Context: controller irqlocked, endpoint selected
  */
 static int cppi_channel_abort(struct dma_channel *channel)
 {
     struct cppi_channel *cppi_ch;
     struct cppi     *controller;
     void __iomem        *mbase;
     void __iomem        *tibase;
     void __iomem        *regs;
     u32         value;
     struct cppi_descriptor  *queue;
 
     cppi_ch = container_of(channel, struct cppi_channel, channel);
 
     controller = cppi_ch->controller;
 
     switch (channel->status) {
     case MUSB_DMA_STATUS_BUS_ABORT:
     case MUSB_DMA_STATUS_CORE_ABORT:
         /* from RX or TX fault irq handler */
     case MUSB_DMA_STATUS_BUSY:
         /* the hardware needs shutting down */
         regs = cppi_ch->hw_ep->regs;
         break;
     case MUSB_DMA_STATUS_UNKNOWN:
     case MUSB_DMA_STATUS_FREE:
         return 0;
     default:
         return -EINVAL;
     }
 
     if (!cppi_ch->transmit && cppi_ch->head)
         cppi_dump_rxq(3, "/abort", cppi_ch);
 
     mbase = controller->mregs;
     tibase = controller->tibase;
 
     queue = cppi_ch->head;
     cppi_ch->head = NULL;
     cppi_ch->tail = NULL;
 
     /* REVISIT should rely on caller having done this,
      * and caller should rely on us not changing it.
      * peripheral code is safe ... check host too.
      */
     musb_ep_select(mbase, cppi_ch->index + 1);
 
     if (cppi_ch->transmit) {
         struct cppi_tx_stateram __iomem *tx_ram;
         /* REVISIT put timeouts on these controller handshakes */
 
         cppi_dump_tx(6, cppi_ch, " (teardown)");
 
         /* teardown DMA engine then usb core */
         do {
             value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG);
         } while (!(value & CPPI_TEAR_READY));
         musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index);
 
         tx_ram = cppi_ch->state_ram;
         do {
             value = musb_readl(&tx_ram->tx_complete, 0);
         } while (0xFFFFFFFC != value);
 
         /* FIXME clean up the transfer state ... here?
          * the completion routine should get called with
          * an appropriate status code.
          */
 
         value = musb_readw(regs, MUSB_TXCSR);
         value &= ~MUSB_TXCSR_DMAENAB;
         value |= MUSB_TXCSR_FLUSHFIFO;
         musb_writew(regs, MUSB_TXCSR, value);
         musb_writew(regs, MUSB_TXCSR, value);
 
         /*
          * 1. Write to completion Ptr value 0x1(bit 0 set)
          *    (write back mode)
          * 2. Wait for abort interrupt and then put the channel in
          *    compare mode by writing 1 to the tx_complete register.
          */
         cppi_reset_tx(tx_ram, 1);
         cppi_ch->head = NULL;
         musb_writel(&tx_ram->tx_complete, 0, 1);
         cppi_dump_tx(5, cppi_ch, " (done teardown)");
 
         /* REVISIT tx side _should_ clean up the same way
          * as the RX side ... this does no cleanup at all!
          */
 
     } else /* RX */ {
         u16         csr;
 
         /* NOTE: docs don't guarantee any of this works ...  we
          * expect that if the usb core stops telling the cppi core
          * to pull more data from it, then it'll be safe to flush
          * current RX DMA state iff any pending fifo transfer is done.
          */
 
         core_rxirq_disable(tibase, cppi_ch->index + 1);
 
         /* for host, ensure ReqPkt is never set again */
         if (is_host_active(cppi_ch->controller->controller.musb)) {
             value = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
             value &= ~((0x3) << (cppi_ch->index * 2));
             musb_writel(tibase, DAVINCI_AUTOREQ_REG, value);
         }
 
         csr = musb_readw(regs, MUSB_RXCSR);
 
         /* for host, clear (just) ReqPkt at end of current packet(s) */
         if (is_host_active(cppi_ch->controller->controller.musb)) {
             csr |= MUSB_RXCSR_H_WZC_BITS;
             csr &= ~MUSB_RXCSR_H_REQPKT;
         } else
             csr |= MUSB_RXCSR_P_WZC_BITS;
 
         /* clear dma enable */
         csr &= ~(MUSB_RXCSR_DMAENAB);
         musb_writew(regs, MUSB_RXCSR, csr);
         csr = musb_readw(regs, MUSB_RXCSR);
 
         /* Quiesce: wait for current dma to finish (if not cleanup).
          * We can't use bit zero of stateram->rx_sop, since that
          * refers to an entire "DMA packet" not just emptying the
          * current fifo.  Most segments need multiple usb packets.
          */
         if (channel->status == MUSB_DMA_STATUS_BUSY)
             udelay(50);
 
         /* scan the current list, reporting any data that was
          * transferred and acking any IRQ
          */
         cppi_rx_scan(controller, cppi_ch->index);
 
         /* clobber the existing state once it's idle
          *
          * NOTE:  arguably, we should also wait for all the other
          * RX channels to quiesce (how??) and then temporarily
          * disable RXCPPI_CTRL_REG ... but it seems that we can
          * rely on the controller restarting from state ram, with
          * only RXCPPI_BUFCNT state being bogus.  BUFCNT will
          * correct itself after the next DMA transfer though.
          *
          * REVISIT does using rndis mode change that?
          */
         cppi_reset_rx(cppi_ch->state_ram);
 
         /* next DMA request _should_ load cppi head ptr */
 
         /* ... we don't "free" that list, only mutate it in place.  */
         cppi_dump_rx(5, cppi_ch, " (done abort)");
 
         /* clean up previously pending bds */
         cppi_bd_free(cppi_ch, cppi_ch->last_processed);
         cppi_ch->last_processed = NULL;
 
         while (queue) {
             struct cppi_descriptor  *tmp = queue->next;
 
             cppi_bd_free(cppi_ch, queue);
             queue = tmp;
         }
     }
 
     channel->status = MUSB_DMA_STATUS_FREE;
     cppi_ch->buf_dma = 0;
     cppi_ch->offset = 0;
     cppi_ch->buf_len = 0;
     cppi_ch->maxpacket = 0;
     return 0;
 }
 
 /* TBD Queries:
  *
  * Power Management ... probably turn off cppi during suspend, restart;
  * check state ram?  Clocking is presumably shared with usb core.
  */

This page was automatically generated by the 2.1.0 LXR engine. The LXR team