OSCL-LXR linux-6.0.1/​drivers/​usb/​host/​octeon-hcd.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0
 /*
  * This file is subject to the terms and conditions of the GNU General Public
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
  * Copyright (C) 2008 Cavium Networks
  *
  * Some parts of the code were originally released under BSD license:
  *
  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  * reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
  * met:
  *
  *   * Redistributions of source code must retain the above copyright
  *     notice, this list of conditions and the following disclaimer.
  *
  *   * Redistributions in binary form must reproduce the above
  *     copyright notice, this list of conditions and the following
  *     disclaimer in the documentation and/or other materials provided
  *     with the distribution.
  *
  *   * Neither the name of Cavium Networks nor the names of
  *     its contributors may be used to endorse or promote products
  *     derived from this software without specific prior written
  *     permission.
  *
  * This Software, including technical data, may be subject to U.S. export
  * control laws, including the U.S. Export Administration Act and its associated
  * regulations, and may be subject to export or import regulations in other
  * countries.
  *
  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  */
 
 #include <linux/usb.h>
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/usb/hcd.h>
 #include <linux/prefetch.h>
 #include <linux/irqdomain.h>
 #include <linux/dma-mapping.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 
 #include <asm/octeon/octeon.h>
 
 #include "octeon-hcd.h"
 
 /**
  * enum cvmx_usb_speed - the possible USB device speeds
  *
  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  */
 enum cvmx_usb_speed {
     CVMX_USB_SPEED_HIGH = 0,
     CVMX_USB_SPEED_FULL = 1,
     CVMX_USB_SPEED_LOW = 2,
 };
 
 /**
  * enum cvmx_usb_transfer - the possible USB transfer types
  *
  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  *                 transfers
  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  *                 priority periodic transfers
  * @CVMX_USB_TRANSFER_BULK:    USB transfer type bulk for large low priority
  *                 transfers
  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  *                 periodic transfers
  */
 enum cvmx_usb_transfer {
     CVMX_USB_TRANSFER_CONTROL = 0,
     CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
     CVMX_USB_TRANSFER_BULK = 2,
     CVMX_USB_TRANSFER_INTERRUPT = 3,
 };
 
 /**
  * enum cvmx_usb_direction - the transfer directions
  *
  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
  */
 enum cvmx_usb_direction {
     CVMX_USB_DIRECTION_OUT,
     CVMX_USB_DIRECTION_IN,
 };
 
 /**
  * enum cvmx_usb_status - possible callback function status codes
  *
  * @CVMX_USB_STATUS_OK:       The transaction / operation finished without
  *                any errors
  * @CVMX_USB_STATUS_SHORT:    FIXME: This is currently not implemented
  * @CVMX_USB_STATUS_CANCEL:   The transaction was canceled while in flight
  *                by a user call to cvmx_usb_cancel
  * @CVMX_USB_STATUS_ERROR:    The transaction aborted with an unexpected
  *                error status
  * @CVMX_USB_STATUS_STALL:    The transaction received a USB STALL response
  *                from the device
  * @CVMX_USB_STATUS_XACTERR:      The transaction failed with an error from the
  *                device even after a number of retries
  * @CVMX_USB_STATUS_DATATGLERR:   The transaction failed with a data toggle
  *                error even after a number of retries
  * @CVMX_USB_STATUS_BABBLEERR:    The transaction failed with a babble error
  * @CVMX_USB_STATUS_FRAMEERR:     The transaction failed with a frame error
  *                even after a number of retries
  */
 enum cvmx_usb_status {
     CVMX_USB_STATUS_OK,
     CVMX_USB_STATUS_SHORT,
     CVMX_USB_STATUS_CANCEL,
     CVMX_USB_STATUS_ERROR,
     CVMX_USB_STATUS_STALL,
     CVMX_USB_STATUS_XACTERR,
     CVMX_USB_STATUS_DATATGLERR,
     CVMX_USB_STATUS_BABBLEERR,
     CVMX_USB_STATUS_FRAMEERR,
 };
 
 /**
  * struct cvmx_usb_port_status - the USB port status information
  *
  * @port_enabled:   1 = Usb port is enabled, 0 = disabled
  * @port_over_current:  1 = Over current detected, 0 = Over current not
  *          detected. Octeon doesn't support over current detection.
  * @port_powered:   1 = Port power is being supplied to the device, 0 =
  *          power is off. Octeon doesn't support turning port power
  *          off.
  * @port_speed:     Current port speed.
  * @connected:      1 = A device is connected to the port, 0 = No device is
  *          connected.
  * @connect_change: 1 = Device connected state changed since the last set
  *          status call.
  */
 struct cvmx_usb_port_status {
     u32 reserved            : 25;
     u32 port_enabled        : 1;
     u32 port_over_current       : 1;
     u32 port_powered        : 1;
     enum cvmx_usb_speed port_speed  : 2;
     u32 connected           : 1;
     u32 connect_change      : 1;
 };
 
 /**
  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
  *
  * @offset: This is the offset in bytes into the main buffer where this data
  *      is stored.
  * @length: This is the length in bytes of the data.
  * @status: This is the status of this individual packet transfer.
  */
 struct cvmx_usb_iso_packet {
     int offset;
     int length;
     enum cvmx_usb_status status;
 };
 
 /**
  * enum cvmx_usb_initialize_flags - flags used by the initialization function
  *
  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
  *                        as clock source at USB_XO and
  *                        USB_XI.
  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
  *                        board clock source at USB_XO.
  *                        USB_XI should be tied to GND.
  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
  *                        crystal
  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
  *                        data transfer use for the USB
  */
 enum cvmx_usb_initialize_flags {
     CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI       = 1 << 0,
     CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND      = 1 << 1,
     CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK    = 3 << 3,
     CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ       = 1 << 3,
     CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ       = 2 << 3,
     CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ       = 3 << 3,
     /* Bits 3-4 used to encode the clock frequency */
     CVMX_USB_INITIALIZE_FLAGS_NO_DMA        = 1 << 5,
 };
 
 /**
  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
  *
  * @CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
  *                 actively using hardware.
  * @CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high speed
  *                 pipe is in the ping state.
  */
 enum cvmx_usb_pipe_flags {
     CVMX_USB_PIPE_FLAGS_SCHEDULED   = 1 << 17,
     CVMX_USB_PIPE_FLAGS_NEED_PING   = 1 << 18,
 };
 
 /* Maximum number of times to retry failed transactions */
 #define MAX_RETRIES     3
 
 /* Maximum number of hardware channels supported by the USB block */
 #define MAX_CHANNELS        8
 
 /*
  * The low level hardware can transfer a maximum of this number of bytes in each
  * transfer. The field is 19 bits wide
  */
 #define MAX_TRANSFER_BYTES  ((1 << 19) - 1)
 
 /*
  * The low level hardware can transfer a maximum of this number of packets in
  * each transfer. The field is 10 bits wide
  */
 #define MAX_TRANSFER_PACKETS    ((1 << 10) - 1)
 
 /**
  * Logical transactions may take numerous low level
  * transactions, especially when splits are concerned. This
  * enum represents all of the possible stages a transaction can
  * be in. Note that split completes are always even. This is so
  * the NAK handler can backup to the previous low level
  * transaction with a simple clearing of bit 0.
  */
 enum cvmx_usb_stage {
     CVMX_USB_STAGE_NON_CONTROL,
     CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
     CVMX_USB_STAGE_SETUP,
     CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
     CVMX_USB_STAGE_DATA,
     CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
     CVMX_USB_STAGE_STATUS,
     CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 };
 
 /**
  * struct cvmx_usb_transaction - describes each pending USB transaction
  *               regardless of type. These are linked together
  *               to form a list of pending requests for a pipe.
  *
  * @node:       List node for transactions in the pipe.
  * @type:       Type of transaction, duplicated of the pipe.
  * @flags:      State flags for this transaction.
  * @buffer:     User's physical buffer address to read/write.
  * @buffer_length:  Size of the user's buffer in bytes.
  * @control_header: For control transactions, physical address of the 8
  *          byte standard header.
  * @iso_start_frame:    For ISO transactions, the starting frame number.
  * @iso_number_packets: For ISO transactions, the number of packets in the
  *          request.
  * @iso_packets:    For ISO transactions, the sub packets in the request.
  * @actual_bytes:   Actual bytes transfer for this transaction.
  * @stage:      For control transactions, the current stage.
  * @urb:        URB.
  */
 struct cvmx_usb_transaction {
     struct list_head node;
     enum cvmx_usb_transfer type;
     u64 buffer;
     int buffer_length;
     u64 control_header;
     int iso_start_frame;
     int iso_number_packets;
     struct cvmx_usb_iso_packet *iso_packets;
     int xfersize;
     int pktcnt;
     int retries;
     int actual_bytes;
     enum cvmx_usb_stage stage;
     struct urb *urb;
 };
 
 /**
  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
  *            and some USB device. It contains a list of pending
  *            request to the device.
  *
  * @node:       List node for pipe list
  * @next:       Pipe after this one in the list
  * @transactions:   List of pending transactions
  * @interval:       For periodic pipes, the interval between packets in
  *          frames
  * @next_tx_frame:  The next frame this pipe is allowed to transmit on
  * @flags:      State flags for this pipe
  * @device_speed:   Speed of device connected to this pipe
  * @transfer_type:  Type of transaction supported by this pipe
  * @transfer_dir:   IN or OUT. Ignored for Control
  * @multi_count:    Max packet in a row for the device
  * @max_packet:     The device's maximum packet size in bytes
  * @device_addr:    USB device address at other end of pipe
  * @endpoint_num:   USB endpoint number at other end of pipe
  * @hub_device_addr:    Hub address this device is connected to
  * @hub_port:       Hub port this device is connected to
  * @pid_toggle:     This toggles between 0/1 on every packet send to track
  *          the data pid needed
  * @channel:        Hardware DMA channel for this pipe
  * @split_sc_frame: The low order bits of the frame number the split
  *          complete should be sent on
  */
 struct cvmx_usb_pipe {
     struct list_head node;
     struct list_head transactions;
     u64 interval;
     u64 next_tx_frame;
     enum cvmx_usb_pipe_flags flags;
     enum cvmx_usb_speed device_speed;
     enum cvmx_usb_transfer transfer_type;
     enum cvmx_usb_direction transfer_dir;
     int multi_count;
     u16 max_packet;
     u8 device_addr;
     u8 endpoint_num;
     u8 hub_device_addr;
     u8 hub_port;
     u8 pid_toggle;
     u8 channel;
     s8 split_sc_frame;
 };
 
 struct cvmx_usb_tx_fifo {
     struct {
         int channel;
         int size;
         u64 address;
     } entry[MAX_CHANNELS + 1];
     int head;
     int tail;
 };
 
 /**
  * struct octeon_hcd - the state of the USB block
  *
  * lock:           Serialization lock.
  * init_flags:         Flags passed to initialize.
  * index:          Which USB block this is for.
  * idle_hardware_channels: Bit set for every idle hardware channel.
  * usbcx_hprt:         Stored port status so we don't need to read a CSR to
  *             determine splits.
  * pipe_for_channel:       Map channels to pipes.
  * pipe:           Storage for pipes.
  * indent:         Used by debug output to indent functions.
  * port_status:        Last port status used for change notification.
  * idle_pipes:         List of open pipes that have no transactions.
  * active_pipes:       Active pipes indexed by transfer type.
  * frame_number:       Increments every SOF interrupt for time keeping.
  * active_split:       Points to the current active split, or NULL.
  */
 struct octeon_hcd {
     spinlock_t lock; /* serialization lock */
     int init_flags;
     int index;
     int idle_hardware_channels;
     union cvmx_usbcx_hprt usbcx_hprt;
     struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
     int indent;
     struct cvmx_usb_port_status port_status;
     struct list_head idle_pipes;
     struct list_head active_pipes[4];
     u64 frame_number;
     struct cvmx_usb_transaction *active_split;
     struct cvmx_usb_tx_fifo periodic;
     struct cvmx_usb_tx_fifo nonperiodic;
 };
 
 /*
  * This macro logically sets a single field in a CSR. It does the sequence
  * read, modify, and write
  */
 #define USB_SET_FIELD32(address, _union, field, value)      \
     do {                            \
         union _union c;                 \
                                 \
         c.u32 = cvmx_usb_read_csr32(usb, address);  \
         c.s.field = value;              \
         cvmx_usb_write_csr32(usb, address, c.u32);  \
     } while (0)
 
 /* Returns the IO address to push/pop stuff data from the FIFOs */
 #define USB_FIFO_ADDRESS(channel, usb_index) \
     (CVMX_USBCX_GOTGCTL(usb_index) + ((channel) + 1) * 0x1000)
 
 /**
  * struct octeon_temp_buffer - a bounce buffer for USB transfers
  * @orig_buffer: the original buffer passed by the USB stack
  * @data:    the newly allocated temporary buffer (excluding meta-data)
  *
  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
  * the buffer is too short, we need to allocate a temporary one, and this struct
  * represents it.
  */
 struct octeon_temp_buffer {
     void *orig_buffer;
     u8 data[];
 };
 
 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
 {
     return container_of((void *)p, struct usb_hcd, hcd_priv);
 }
 
 /**
  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
  *                            (if needed)
  * @urb:    URB.
  * @mem_flags:  Memory allocation flags.
  *
  * This function allocates a temporary bounce buffer whenever it's needed
  * due to HW limitations.
  */
 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 {
     struct octeon_temp_buffer *temp;
 
     if (urb->num_sgs || urb->sg ||
         (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
         !(urb->transfer_buffer_length % sizeof(u32)))
         return 0;
 
     temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
                sizeof(*temp), mem_flags);
     if (!temp)
         return -ENOMEM;
 
     temp->orig_buffer = urb->transfer_buffer;
     if (usb_urb_dir_out(urb))
         memcpy(temp->data, urb->transfer_buffer,
                urb->transfer_buffer_length);
     urb->transfer_buffer = temp->data;
     urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 
     return 0;
 }
 
 /**
  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
  * @urb: URB.
  *
  * Frees a buffer allocated by octeon_alloc_temp_buffer().
  */
 static void octeon_free_temp_buffer(struct urb *urb)
 {
     struct octeon_temp_buffer *temp;
     size_t length;
 
     if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
         return;
 
     temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
                 data);
     if (usb_urb_dir_in(urb)) {
         if (usb_pipeisoc(urb->pipe))
             length = urb->transfer_buffer_length;
         else
             length = urb->actual_length;
 
         memcpy(temp->orig_buffer, urb->transfer_buffer, length);
     }
     urb->transfer_buffer = temp->orig_buffer;
     urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
     kfree(temp);
 }
 
 /**
  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
  * @hcd:    USB HCD structure.
  * @urb:    URB.
  * @mem_flags:  Memory allocation flags.
  */
 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
                   gfp_t mem_flags)
 {
     int ret;
 
     ret = octeon_alloc_temp_buffer(urb, mem_flags);
     if (ret)
         return ret;
 
     ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
     if (ret)
         octeon_free_temp_buffer(urb);
 
     return ret;
 }
 
 /**
  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
  * @hcd:    USB HCD structure.
  * @urb:    URB.
  */
 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 {
     usb_hcd_unmap_urb_for_dma(hcd, urb);
     octeon_free_temp_buffer(urb);
 }
 
 /**
  * Read a USB 32bit CSR. It performs the necessary address swizzle
  * for 32bit CSRs and logs the value in a readable format if
  * debugging is on.
  *
  * @usb:     USB block this access is for
  * @address: 64bit address to read
  *
  * Returns: Result of the read
  */
 static inline u32 cvmx_usb_read_csr32(struct octeon_hcd *usb, u64 address)
 {
     return cvmx_read64_uint32(address ^ 4);
 }
 
 /**
  * Write a USB 32bit CSR. It performs the necessary address
  * swizzle for 32bit CSRs and logs the value in a readable format
  * if debugging is on.
  *
  * @usb:     USB block this access is for
  * @address: 64bit address to write
  * @value:   Value to write
  */
 static inline void cvmx_usb_write_csr32(struct octeon_hcd *usb,
                     u64 address, u32 value)
 {
     cvmx_write64_uint32(address ^ 4, value);
     cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 }
 
 /**
  * Return non zero if this pipe connects to a non HIGH speed
  * device through a high speed hub.
  *
  * @usb:    USB block this access is for
  * @pipe:   Pipe to check
  *
  * Returns: Non zero if we need to do split transactions
  */
 static inline int cvmx_usb_pipe_needs_split(struct octeon_hcd *usb,
                         struct cvmx_usb_pipe *pipe)
 {
     return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
            usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 }
 
 /**
  * Trivial utility function to return the correct PID for a pipe
  *
  * @pipe:   pipe to check
  *
  * Returns: PID for pipe
  */
 static inline int cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 {
     if (pipe->pid_toggle)
         return 2; /* Data1 */
     return 0; /* Data0 */
 }
 
 /* Loops through register until txfflsh or rxfflsh become zero.*/
 static int cvmx_wait_tx_rx(struct octeon_hcd *usb, int fflsh_type)
 {
     int result;
     u64 address = CVMX_USBCX_GRSTCTL(usb->index);
     u64 done = cvmx_get_cycle() + 100 *
            (u64)octeon_get_clock_rate / 1000000;
     union cvmx_usbcx_grstctl c;
 
     while (1) {
         c.u32 = cvmx_usb_read_csr32(usb, address);
         if (fflsh_type == 0 && c.s.txfflsh == 0) {
             result = 0;
             break;
         } else if (fflsh_type == 1 && c.s.rxfflsh == 0) {
             result = 0;
             break;
         } else if (cvmx_get_cycle() > done) {
             result = -1;
             break;
         }
 
         __delay(100);
     }
     return result;
 }
 
 static void cvmx_fifo_setup(struct octeon_hcd *usb)
 {
     union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
     union cvmx_usbcx_gnptxfsiz npsiz;
     union cvmx_usbcx_hptxfsiz psiz;
 
     usbcx_ghwcfg3.u32 = cvmx_usb_read_csr32(usb,
                         CVMX_USBCX_GHWCFG3(usb->index));
 
     /*
      * Program the USBC_GRXFSIZ register to select the size of the receive
      * FIFO (25%).
      */
     USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), cvmx_usbcx_grxfsiz,
             rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
 
     /*
      * Program the USBC_GNPTXFSIZ register to select the size and the start
      * address of the non-periodic transmit FIFO for nonperiodic
      * transactions (50%).
      */
     npsiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
     npsiz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
     npsiz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
     cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), npsiz.u32);
 
     /*
      * Program the USBC_HPTXFSIZ register to select the size and start
      * address of the periodic transmit FIFO for periodic transactions
      * (25%).
      */
     psiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
     psiz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
     psiz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
     cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), psiz.u32);
 
     /* Flush all FIFOs */
     USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
             cvmx_usbcx_grstctl, txfnum, 0x10);
     USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
             cvmx_usbcx_grstctl, txfflsh, 1);
     cvmx_wait_tx_rx(usb, 0);
     USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
             cvmx_usbcx_grstctl, rxfflsh, 1);
     cvmx_wait_tx_rx(usb, 1);
 }
 
 /**
  * Shutdown a USB port after a call to cvmx_usb_initialize().
  * The port should be disabled with all pipes closed when this
  * function is called.
  *
  * @usb: USB device state populated by cvmx_usb_initialize().
  *
  * Returns: 0 or a negative error code.
  */
 static int cvmx_usb_shutdown(struct octeon_hcd *usb)
 {
     union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 
     /* Make sure all pipes are closed */
     if (!list_empty(&usb->idle_pipes) ||
         !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
         !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
         !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
         !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
         return -EBUSY;
 
     /* Disable the clocks and put them in power on reset */
     usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
     usbn_clk_ctl.s.enable = 1;
     usbn_clk_ctl.s.por = 1;
     usbn_clk_ctl.s.hclk_rst = 1;
     usbn_clk_ctl.s.prst = 0;
     usbn_clk_ctl.s.hrst = 0;
     cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
     return 0;
 }
 
 /**
  * Initialize a USB port for use. This must be called before any
  * other access to the Octeon USB port is made. The port starts
  * off in the disabled state.
  *
  * @dev:     Pointer to struct device for logging purposes.
  * @usb:     Pointer to struct octeon_hcd.
  *
  * Returns: 0 or a negative error code.
  */
 static int cvmx_usb_initialize(struct device *dev,
                    struct octeon_hcd *usb)
 {
     int channel;
     int divisor;
     int retries = 0;
     union cvmx_usbcx_hcfg usbcx_hcfg;
     union cvmx_usbnx_clk_ctl usbn_clk_ctl;
     union cvmx_usbcx_gintsts usbc_gintsts;
     union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
     union cvmx_usbcx_gintmsk usbcx_gintmsk;
     union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
     union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 
 retry:
     /*
      * Power On Reset and PHY Initialization
      *
      * 1. Wait for DCOK to assert (nothing to do)
      *
      * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
      *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
      */
     usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
     usbn_clk_ctl.s.por = 1;
     usbn_clk_ctl.s.hrst = 0;
     usbn_clk_ctl.s.prst = 0;
     usbn_clk_ctl.s.hclk_rst = 0;
     usbn_clk_ctl.s.enable = 0;
     /*
      * 2b. Select the USB reference clock/crystal parameters by writing
      *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
      */
     if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
         /*
          * The USB port uses 12/24/48MHz 2.5V board clock
          * source at USB_XO. USB_XI should be tied to GND.
          * Most Octeon evaluation boards require this setting
          */
         if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
             OCTEON_IS_MODEL(OCTEON_CN56XX) ||
             OCTEON_IS_MODEL(OCTEON_CN50XX))
             /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
             usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
         else
             /* From CN52XX manual */
             usbn_clk_ctl.s.p_rtype = 1;
 
         switch (usb->init_flags &
             CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
         case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
             usbn_clk_ctl.s.p_c_sel = 0;
             break;
         case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
             usbn_clk_ctl.s.p_c_sel = 1;
             break;
         case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
             usbn_clk_ctl.s.p_c_sel = 2;
             break;
         }
     } else {
         /*
          * The USB port uses a 12MHz crystal as clock source
          * at USB_XO and USB_XI
          */
         if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
             /* From CN31XX,CN30XX manual */
             usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
         else
             /* From CN56XX,CN52XX,CN50XX manuals. */
             usbn_clk_ctl.s.p_rtype = 0;
 
         usbn_clk_ctl.s.p_c_sel = 0;
     }
     /*
      * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
      *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
      *     such that USB is as close as possible to 125Mhz
      */
     divisor = DIV_ROUND_UP(octeon_get_clock_rate(), 125000000);
     /* Lower than 4 doesn't seem to work properly */
     if (divisor < 4)
         divisor = 4;
     usbn_clk_ctl.s.divide = divisor;
     usbn_clk_ctl.s.divide2 = 0;
     cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 
     /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
     usbn_clk_ctl.s.hclk_rst = 1;
     cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
     /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
     __delay(64);
     /*
      * 3. Program the power-on reset field in the USBN clock-control
      *    register:
      *    USBN_CLK_CTL[POR] = 0
      */
     usbn_clk_ctl.s.por = 0;
     cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
     /* 4. Wait 1 ms for PHY clock to start */
     mdelay(1);
     /*
      * 5. Program the Reset input from automatic test equipment field in the
      *    USBP control and status register:
      *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
      */
     usbn_usbp_ctl_status.u64 =
         cvmx_read64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index));
     usbn_usbp_ctl_status.s.ate_reset = 1;
     cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
                 usbn_usbp_ctl_status.u64);
     /* 6. Wait 10 cycles */
     __delay(10);
     /*
      * 7. Clear ATE_RESET field in the USBN clock-control register:
      *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
      */
     usbn_usbp_ctl_status.s.ate_reset = 0;
     cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
                 usbn_usbp_ctl_status.u64);
     /*
      * 8. Program the PHY reset field in the USBN clock-control register:
      *    USBN_CLK_CTL[PRST] = 1
      */
     usbn_clk_ctl.s.prst = 1;
     cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
     /*
      * 9. Program the USBP control and status register to select host or
      *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
      *    device
      */
     usbn_usbp_ctl_status.s.hst_mode = 0;
     cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
                 usbn_usbp_ctl_status.u64);
     /* 10. Wait 1 us */
     udelay(1);
     /*
      * 11. Program the hreset_n field in the USBN clock-control register:
      *     USBN_CLK_CTL[HRST] = 1
      */
     usbn_clk_ctl.s.hrst = 1;
     cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
     /* 12. Proceed to USB core initialization */
     usbn_clk_ctl.s.enable = 1;
     cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
     udelay(1);
 
     /*
      * USB Core Initialization
      *
      * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
      *    determine USB core configuration parameters.
      *
      *    Nothing needed
      *
      * 2. Program the following fields in the global AHB configuration
      *    register (USBC_GAHBCFG)
      *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
      *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
      *    Nonperiodic TxFIFO empty level (slave mode only),
      *    USBC_GAHBCFG[NPTXFEMPLVL]
      *    Periodic TxFIFO empty level (slave mode only),
      *    USBC_GAHBCFG[PTXFEMPLVL]
      *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
      */
     usbcx_gahbcfg.u32 = 0;
     usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
                   CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
     usbcx_gahbcfg.s.hbstlen = 0;
     usbcx_gahbcfg.s.nptxfemplvl = 1;
     usbcx_gahbcfg.s.ptxfemplvl = 1;
     usbcx_gahbcfg.s.glblintrmsk = 1;
     cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
                  usbcx_gahbcfg.u32);
 
     /*
      * 3. Program the following fields in USBC_GUSBCFG register.
      *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
      *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
      *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
      *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
      */
     usbcx_gusbcfg.u32 = cvmx_usb_read_csr32(usb,
                         CVMX_USBCX_GUSBCFG(usb->index));
     usbcx_gusbcfg.s.toutcal = 0;
     usbcx_gusbcfg.s.ddrsel = 0;
     usbcx_gusbcfg.s.usbtrdtim = 0x5;
     usbcx_gusbcfg.s.phylpwrclksel = 0;
     cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
                  usbcx_gusbcfg.u32);
 
     /*
      * 4. The software must unmask the following bits in the USBC_GINTMSK
      *    register.
      *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
      *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
      */
     usbcx_gintmsk.u32 = cvmx_usb_read_csr32(usb,
                         CVMX_USBCX_GINTMSK(usb->index));
     usbcx_gintmsk.s.otgintmsk = 1;
     usbcx_gintmsk.s.modemismsk = 1;
     usbcx_gintmsk.s.hchintmsk = 1;
     usbcx_gintmsk.s.sofmsk = 0;
     /* We need RX FIFO interrupts if we don't have DMA */
     if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
         usbcx_gintmsk.s.rxflvlmsk = 1;
     cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
                  usbcx_gintmsk.u32);
 
     /*
      * Disable all channel interrupts. We'll enable them per channel later.
      */
     for (channel = 0; channel < 8; channel++)
         cvmx_usb_write_csr32(usb,
                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
                      0);
 
     /*
      * Host Port Initialization
      *
      * 1. Program the host-port interrupt-mask field to unmask,
      *    USBC_GINTMSK[PRTINT] = 1
      */
     USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
             cvmx_usbcx_gintmsk, prtintmsk, 1);
     USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
             cvmx_usbcx_gintmsk, disconnintmsk, 1);
 
     /*
      * 2. Program the USBC_HCFG register to select full-speed host
      *    or high-speed host.
      */
     usbcx_hcfg.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
     usbcx_hcfg.s.fslssupp = 0;
     usbcx_hcfg.s.fslspclksel = 0;
     cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 
     cvmx_fifo_setup(usb);
 
     /*
      * If the controller is getting port events right after the reset, it
      * means the initialization failed. Try resetting the controller again
      * in such case. This is seen to happen after cold boot on DSR-1000N.
      */
     usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
                            CVMX_USBCX_GINTSTS(usb->index));
     cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
                  usbc_gintsts.u32);
     dev_dbg(dev, "gintsts after reset: 0x%x\n", (int)usbc_gintsts.u32);
     if (!usbc_gintsts.s.disconnint && !usbc_gintsts.s.prtint)
         return 0;
     if (retries++ >= 5)
         return -EAGAIN;
     dev_info(dev, "controller reset failed (gintsts=0x%x) - retrying\n",
          (int)usbc_gintsts.u32);
     msleep(50);
     cvmx_usb_shutdown(usb);
     msleep(50);
     goto retry;
 }
 
 /**
  * Reset a USB port. After this call succeeds, the USB port is
  * online and servicing requests.
  *
  * @usb: USB device state populated by cvmx_usb_initialize().
  */
 static void cvmx_usb_reset_port(struct octeon_hcd *usb)
 {
     usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
                           CVMX_USBCX_HPRT(usb->index));
 
     /* Program the port reset bit to start the reset process */
     USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
             prtrst, 1);
 
     /*
      * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
      * process to complete.
      */
     mdelay(50);
 
     /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
     USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
             prtrst, 0);
 
     /*
      * Read the port speed field to get the enumerated speed,
      * USBC_HPRT[PRTSPD].
      */
     usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
                           CVMX_USBCX_HPRT(usb->index));
 }
 
 /**
  * Disable a USB port. After this call the USB port will not
  * generate data transfers and will not generate events.
  * Transactions in process will fail and call their
  * associated callbacks.
  *
  * @usb: USB device state populated by cvmx_usb_initialize().
  *
  * Returns: 0 or a negative error code.
  */
 static int cvmx_usb_disable(struct octeon_hcd *usb)
 {
     /* Disable the port */
     USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
             prtena, 1);
     return 0;
 }
 
 /**
  * Get the current state of the USB port. Use this call to
  * determine if the usb port has anything connected, is enabled,
  * or has some sort of error condition. The return value of this
  * call has "changed" bits to signal of the value of some fields
  * have changed between calls.
  *
  * @usb: USB device state populated by cvmx_usb_initialize().
  *
  * Returns: Port status information
  */
 static struct cvmx_usb_port_status cvmx_usb_get_status(struct octeon_hcd *usb)
 {
     union cvmx_usbcx_hprt usbc_hprt;
     struct cvmx_usb_port_status result;
 
     memset(&result, 0, sizeof(result));
 
     usbc_hprt.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
     result.port_enabled = usbc_hprt.s.prtena;
     result.port_over_current = usbc_hprt.s.prtovrcurract;
     result.port_powered = usbc_hprt.s.prtpwr;
     result.port_speed = usbc_hprt.s.prtspd;
     result.connected = usbc_hprt.s.prtconnsts;
     result.connect_change =
         result.connected != usb->port_status.connected;
 
     return result;
 }
 
 /**
  * Open a virtual pipe between the host and a USB device. A pipe
  * must be opened before data can be transferred between a device
  * and Octeon.
  *
  * @usb:         USB device state populated by cvmx_usb_initialize().
  * @device_addr:
  *           USB device address to open the pipe to
  *           (0-127).
  * @endpoint_num:
  *           USB endpoint number to open the pipe to
  *           (0-15).
  * @device_speed:
  *           The speed of the device the pipe is going
  *           to. This must match the device's speed,
  *           which may be different than the port speed.
  * @max_packet:      The maximum packet length the device can
  *           transmit/receive (low speed=0-8, full
  *           speed=0-1023, high speed=0-1024). This value
  *           comes from the standard endpoint descriptor
  *           field wMaxPacketSize bits <10:0>.
  * @transfer_type:
  *           The type of transfer this pipe is for.
  * @transfer_dir:
  *           The direction the pipe is in. This is not
  *           used for control pipes.
  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
  *           this is how often the transfer is scheduled
  *           for. All other transfers should specify
  *           zero. The units are in frames (8000/sec at
  *           high speed, 1000/sec for full speed).
  * @multi_count:
  *           For high speed devices, this is the maximum
  *           allowed number of packet per microframe.
  *           Specify zero for non high speed devices. This
  *           value comes from the standard endpoint descriptor
  *           field wMaxPacketSize bits <12:11>.
  * @hub_device_addr:
  *           Hub device address this device is connected
  *           to. Devices connected directly to Octeon
  *           use zero. This is only used when the device
  *           is full/low speed behind a high speed hub.
  *           The address will be of the high speed hub,
  *           not and full speed hubs after it.
  * @hub_port:        Which port on the hub the device is
  *           connected. Use zero for devices connected
  *           directly to Octeon. Like hub_device_addr,
  *           this is only used for full/low speed
  *           devices behind a high speed hub.
  *
  * Returns: A non-NULL value is a pipe. NULL means an error.
  */
 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct octeon_hcd *usb,
                         int device_addr,
                         int endpoint_num,
                         enum cvmx_usb_speed
                             device_speed,
                         int max_packet,
                         enum cvmx_usb_transfer
                             transfer_type,
                         enum cvmx_usb_direction
                             transfer_dir,
                         int interval, int multi_count,
                         int hub_device_addr,
                         int hub_port)
 {
     struct cvmx_usb_pipe *pipe;
 
     pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
     if (!pipe)
         return NULL;
     if ((device_speed == CVMX_USB_SPEED_HIGH) &&
         (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
         (transfer_type == CVMX_USB_TRANSFER_BULK))
         pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
     pipe->device_addr = device_addr;
     pipe->endpoint_num = endpoint_num;
     pipe->device_speed = device_speed;
     pipe->max_packet = max_packet;
     pipe->transfer_type = transfer_type;
     pipe->transfer_dir = transfer_dir;
     INIT_LIST_HEAD(&pipe->transactions);
 
     /*
      * All pipes use interval to rate limit NAK processing. Force an
      * interval if one wasn't supplied
      */
     if (!interval)
         interval = 1;
     if (cvmx_usb_pipe_needs_split(usb, pipe)) {
         pipe->interval = interval * 8;
         /* Force start splits to be schedule on uFrame 0 */
         pipe->next_tx_frame = ((usb->frame_number + 7) & ~7) +
                     pipe->interval;
     } else {
         pipe->interval = interval;
         pipe->next_tx_frame = usb->frame_number + pipe->interval;
     }
     pipe->multi_count = multi_count;
     pipe->hub_device_addr = hub_device_addr;
     pipe->hub_port = hub_port;
     pipe->pid_toggle = 0;
     pipe->split_sc_frame = -1;
     list_add_tail(&pipe->node, &usb->idle_pipes);
 
     /*
      * We don't need to tell the hardware about this pipe yet since
      * it doesn't have any submitted requests
      */
 
     return pipe;
 }
 
 /**
  * Poll the RX FIFOs and remove data as needed. This function is only used
  * in non DMA mode. It is very important that this function be called quickly
  * enough to prevent FIFO overflow.
  *
  * @usb:    USB device state populated by cvmx_usb_initialize().
  */
 static void cvmx_usb_poll_rx_fifo(struct octeon_hcd *usb)
 {
     union cvmx_usbcx_grxstsph rx_status;
     int channel;
     int bytes;
     u64 address;
     u32 *ptr;
 
     rx_status.u32 = cvmx_usb_read_csr32(usb,
                         CVMX_USBCX_GRXSTSPH(usb->index));
     /* Only read data if IN data is there */
     if (rx_status.s.pktsts != 2)
         return;
     /* Check if no data is available */
     if (!rx_status.s.bcnt)
         return;
 
     channel = rx_status.s.chnum;
     bytes = rx_status.s.bcnt;
     if (!bytes)
         return;
 
     /* Get where the DMA engine would have written this data */
     address = cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) +
                      channel * 8);
 
     ptr = cvmx_phys_to_ptr(address);
     cvmx_write64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel * 8,
                 address + bytes);
 
     /* Loop writing the FIFO data for this packet into memory */
     while (bytes > 0) {
         *ptr++ = cvmx_usb_read_csr32(usb,
                     USB_FIFO_ADDRESS(channel, usb->index));
         bytes -= 4;
     }
     CVMX_SYNCW;
 }
 
 /**
  * Fill the TX hardware fifo with data out of the software
  * fifos
  *
  * @usb:        USB device state populated by cvmx_usb_initialize().
  * @fifo:       Software fifo to use
  * @available:      Amount of space in the hardware fifo
  *
  * Returns: Non zero if the hardware fifo was too small and needs
  *      to be serviced again.
  */
 static int cvmx_usb_fill_tx_hw(struct octeon_hcd *usb,
                    struct cvmx_usb_tx_fifo *fifo, int available)
 {
     /*
      * We're done either when there isn't anymore space or the software FIFO
      * is empty
      */
     while (available && (fifo->head != fifo->tail)) {
         int i = fifo->tail;
         const u32 *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
         u64 csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
                            usb->index) ^ 4;
         int words = available;
 
         /* Limit the amount of data to what the SW fifo has */
         if (fifo->entry[i].size <= available) {
             words = fifo->entry[i].size;
             fifo->tail++;
             if (fifo->tail > MAX_CHANNELS)
                 fifo->tail = 0;
         }
 
         /* Update the next locations and counts */
         available -= words;
         fifo->entry[i].address += words * 4;
         fifo->entry[i].size -= words;
 
         /*
          * Write the HW fifo data. The read every three writes is due
          * to an errata on CN3XXX chips
          */
         while (words > 3) {
             cvmx_write64_uint32(csr_address, *ptr++);
             cvmx_write64_uint32(csr_address, *ptr++);
             cvmx_write64_uint32(csr_address, *ptr++);
             cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
             words -= 3;
         }
         cvmx_write64_uint32(csr_address, *ptr++);
         if (--words) {
             cvmx_write64_uint32(csr_address, *ptr++);
             if (--words)
                 cvmx_write64_uint32(csr_address, *ptr++);
         }
         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
     }
     return fifo->head != fifo->tail;
 }
 
 /**
  * Check the hardware FIFOs and fill them as needed
  *
  * @usb:    USB device state populated by cvmx_usb_initialize().
  */
 static void cvmx_usb_poll_tx_fifo(struct octeon_hcd *usb)
 {
     if (usb->periodic.head != usb->periodic.tail) {
         union cvmx_usbcx_hptxsts tx_status;
 
         tx_status.u32 = cvmx_usb_read_csr32(usb,
                             CVMX_USBCX_HPTXSTS(usb->index));
         if (cvmx_usb_fill_tx_hw(usb, &usb->periodic,
                     tx_status.s.ptxfspcavail))
             USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
                     cvmx_usbcx_gintmsk, ptxfempmsk, 1);
         else
             USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
                     cvmx_usbcx_gintmsk, ptxfempmsk, 0);
     }
 
     if (usb->nonperiodic.head != usb->nonperiodic.tail) {
         union cvmx_usbcx_gnptxsts tx_status;
 
         tx_status.u32 = cvmx_usb_read_csr32(usb,
                             CVMX_USBCX_GNPTXSTS(usb->index));
         if (cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
                     tx_status.s.nptxfspcavail))
             USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
                     cvmx_usbcx_gintmsk, nptxfempmsk, 1);
         else
             USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
                     cvmx_usbcx_gintmsk, nptxfempmsk, 0);
     }
 }
 
 /**
  * Fill the TX FIFO with an outgoing packet
  *
  * @usb:      USB device state populated by cvmx_usb_initialize().
  * @channel:      Channel number to get packet from
  */
 static void cvmx_usb_fill_tx_fifo(struct octeon_hcd *usb, int channel)
 {
     union cvmx_usbcx_hccharx hcchar;
     union cvmx_usbcx_hcspltx usbc_hcsplt;
     union cvmx_usbcx_hctsizx usbc_hctsiz;
     struct cvmx_usb_tx_fifo *fifo;
 
     /* We only need to fill data on outbound channels */
     hcchar.u32 = cvmx_usb_read_csr32(usb,
                      CVMX_USBCX_HCCHARX(channel, usb->index));
     if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
         return;
 
     /* OUT Splits only have data on the start and not the complete */
     usbc_hcsplt.u32 = cvmx_usb_read_csr32(usb,
                           CVMX_USBCX_HCSPLTX(channel, usb->index));
     if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
         return;
 
     /*
      * Find out how many bytes we need to fill and convert it into 32bit
      * words.
      */
     usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
                           CVMX_USBCX_HCTSIZX(channel, usb->index));
     if (!usbc_hctsiz.s.xfersize)
         return;
 
     if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
         (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
         fifo = &usb->periodic;
     else
         fifo = &usb->nonperiodic;
 
     fifo->entry[fifo->head].channel = channel;
     fifo->entry[fifo->head].address =
         cvmx_read64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
                    channel * 8);
     fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize + 3) >> 2;
     fifo->head++;
     if (fifo->head > MAX_CHANNELS)
         fifo->head = 0;
 
     cvmx_usb_poll_tx_fifo(usb);
 }
 
 /**
  * Perform channel specific setup for Control transactions. All
  * the generic stuff will already have been done in cvmx_usb_start_channel().
  *
  * @usb:      USB device state populated by cvmx_usb_initialize().
  * @channel:      Channel to setup
  * @pipe:     Pipe for control transaction
  */
 static void cvmx_usb_start_channel_control(struct octeon_hcd *usb,
                        int channel,
                        struct cvmx_usb_pipe *pipe)
 {
     struct usb_hcd *hcd = octeon_to_hcd(usb);
     struct device *dev = hcd->self.controller;
     struct cvmx_usb_transaction *transaction =
         list_first_entry(&pipe->transactions, typeof(*transaction),
                  node);
     struct usb_ctrlrequest *header =
         cvmx_phys_to_ptr(transaction->control_header);
     int bytes_to_transfer = transaction->buffer_length -
         transaction->actual_bytes;
     int packets_to_transfer;
     union cvmx_usbcx_hctsizx usbc_hctsiz;
 
     usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
                           CVMX_USBCX_HCTSIZX(channel, usb->index));
 
     switch (transaction->stage) {
     case CVMX_USB_STAGE_NON_CONTROL:
     case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
         dev_err(dev, "%s: ERROR - Non control stage\n", __func__);
         break;
     case CVMX_USB_STAGE_SETUP:
         usbc_hctsiz.s.pid = 3; /* Setup */
         bytes_to_transfer = sizeof(*header);
         /* All Control operations start with a setup going OUT */
         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
                 cvmx_usbcx_hccharx, epdir,
                 CVMX_USB_DIRECTION_OUT);
         /*
          * Setup send the control header instead of the buffer data. The
          * buffer data will be used in the next stage
          */
         cvmx_write64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
                     channel * 8,
                     transaction->control_header);
         break;
     case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
         usbc_hctsiz.s.pid = 3; /* Setup */
         bytes_to_transfer = 0;
         /* All Control operations start with a setup going OUT */
         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
                 cvmx_usbcx_hccharx, epdir,
                 CVMX_USB_DIRECTION_OUT);
 
         USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
                 cvmx_usbcx_hcspltx, compsplt, 1);
         break;
     case CVMX_USB_STAGE_DATA:
         usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
             if (header->bRequestType & USB_DIR_IN)
                 bytes_to_transfer = 0;
             else if (bytes_to_transfer > pipe->max_packet)
                 bytes_to_transfer = pipe->max_packet;
         }
         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
                 cvmx_usbcx_hccharx, epdir,
                 ((header->bRequestType & USB_DIR_IN) ?
                     CVMX_USB_DIRECTION_IN :
                     CVMX_USB_DIRECTION_OUT));
         break;
     case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
         usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
         if (!(header->bRequestType & USB_DIR_IN))
             bytes_to_transfer = 0;
         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
                 cvmx_usbcx_hccharx, epdir,
                 ((header->bRequestType & USB_DIR_IN) ?
                     CVMX_USB_DIRECTION_IN :
                     CVMX_USB_DIRECTION_OUT));
         USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
                 cvmx_usbcx_hcspltx, compsplt, 1);
         break;
     case CVMX_USB_STAGE_STATUS:
         usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
         bytes_to_transfer = 0;
         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
                 cvmx_usbcx_hccharx, epdir,
                 ((header->bRequestType & USB_DIR_IN) ?
                     CVMX_USB_DIRECTION_OUT :
                     CVMX_USB_DIRECTION_IN));
         break;
     case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
         usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
         bytes_to_transfer = 0;
         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
                 cvmx_usbcx_hccharx, epdir,
                 ((header->bRequestType & USB_DIR_IN) ?
                     CVMX_USB_DIRECTION_OUT :
                     CVMX_USB_DIRECTION_IN));
         USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
                 cvmx_usbcx_hcspltx, compsplt, 1);
         break;
     }
 
     /*
      * Make sure the transfer never exceeds the byte limit of the hardware.
      * Further bytes will be sent as continued transactions
      */
     if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
         /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
         bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
         bytes_to_transfer *= pipe->max_packet;
     }
 
     /*
      * Calculate the number of packets to transfer. If the length is zero
      * we still need to transfer one packet
      */
     packets_to_transfer = DIV_ROUND_UP(bytes_to_transfer,
                        pipe->max_packet);
     if (packets_to_transfer == 0) {
         packets_to_transfer = 1;
     } else if ((packets_to_transfer > 1) &&
             (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
         /*
          * Limit to one packet when not using DMA. Channels must be
          * restarted between every packet for IN transactions, so there
          * is no reason to do multiple packets in a row
          */
         packets_to_transfer = 1;
         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
     } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
         /*
          * Limit the number of packet and data transferred to what the
          * hardware can handle
          */
         packets_to_transfer = MAX_TRANSFER_PACKETS;
         bytes_to_transfer = packets_to_transfer * pipe->max_packet;
     }
 
     usbc_hctsiz.s.xfersize = bytes_to_transfer;
     usbc_hctsiz.s.pktcnt = packets_to_transfer;
 
     cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
                  usbc_hctsiz.u32);
 }
 
 /**
  * Start a channel to perform the pipe's head transaction
  *
  * @usb:      USB device state populated by cvmx_usb_initialize().
  * @channel:      Channel to setup
  * @pipe:     Pipe to start
  */
 static void cvmx_usb_start_channel(struct octeon_hcd *usb, int channel,
                    struct cvmx_usb_pipe *pipe)
 {
     struct cvmx_usb_transaction *transaction =
         list_first_entry(&pipe->transactions, typeof(*transaction),
                  node);
 
     /* Make sure all writes to the DMA region get flushed */
     CVMX_SYNCW;
 
     /* Attach the channel to the pipe */
     usb->pipe_for_channel[channel] = pipe;
     pipe->channel = channel;
     pipe->flags |= CVMX_USB_PIPE_FLAGS_SCHEDULED;
 
     /* Mark this channel as in use */
     usb->idle_hardware_channels &= ~(1 << channel);
 
     /* Enable the channel interrupt bits */
     {
         union cvmx_usbcx_hcintx usbc_hcint;
         union cvmx_usbcx_hcintmskx usbc_hcintmsk;
         union cvmx_usbcx_haintmsk usbc_haintmsk;
 
         /* Clear all channel status bits */
         usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
                              CVMX_USBCX_HCINTX(channel, usb->index));
 
         cvmx_usb_write_csr32(usb,
                      CVMX_USBCX_HCINTX(channel, usb->index),
                      usbc_hcint.u32);
 
         usbc_hcintmsk.u32 = 0;
         usbc_hcintmsk.s.chhltdmsk = 1;
         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
             /*
              * Channels need these extra interrupts when we aren't
              * in DMA mode.
              */
             usbc_hcintmsk.s.datatglerrmsk = 1;
             usbc_hcintmsk.s.frmovrunmsk = 1;
             usbc_hcintmsk.s.bblerrmsk = 1;
             usbc_hcintmsk.s.xacterrmsk = 1;
             if (cvmx_usb_pipe_needs_split(usb, pipe)) {
                 /*
                  * Splits don't generate xfercompl, so we need
                  * ACK and NYET.
                  */
                 usbc_hcintmsk.s.nyetmsk = 1;
                 usbc_hcintmsk.s.ackmsk = 1;
             }
             usbc_hcintmsk.s.nakmsk = 1;
             usbc_hcintmsk.s.stallmsk = 1;
             usbc_hcintmsk.s.xfercomplmsk = 1;
         }
         cvmx_usb_write_csr32(usb,
                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
                      usbc_hcintmsk.u32);
 
         /* Enable the channel interrupt to propagate */
         usbc_haintmsk.u32 = cvmx_usb_read_csr32(usb,
                             CVMX_USBCX_HAINTMSK(usb->index));
         usbc_haintmsk.s.haintmsk |= 1 << channel;
         cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index),
                      usbc_haintmsk.u32);
     }
 
     /* Setup the location the DMA engine uses. */
     {
         u64 reg;
         u64 dma_address = transaction->buffer +
                   transaction->actual_bytes;
 
         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
             dma_address = transaction->buffer +
                     transaction->iso_packets[0].offset +
                     transaction->actual_bytes;
 
         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT)
             reg = CVMX_USBNX_DMA0_OUTB_CHN0(usb->index);
         else
             reg = CVMX_USBNX_DMA0_INB_CHN0(usb->index);
         cvmx_write64_uint64(reg + channel * 8, dma_address);
     }
 
     /* Setup both the size of the transfer and the SPLIT characteristics */
     {
         union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
         union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
         int packets_to_transfer;
         int bytes_to_transfer = transaction->buffer_length -
             transaction->actual_bytes;
 
         /*
          * ISOCHRONOUS transactions store each individual transfer size
          * in the packet structure, not the global buffer_length
          */
         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
             bytes_to_transfer =
                 transaction->iso_packets[0].length -
                 transaction->actual_bytes;
 
         /*
          * We need to do split transactions when we are talking to non
          * high speed devices that are behind a high speed hub
          */
         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
             /*
              * On the start split phase (stage is even) record the
              * frame number we will need to send the split complete.
              * We only store the lower two bits since the time ahead
              * can only be two frames
              */
             if ((transaction->stage & 1) == 0) {
                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
                     pipe->split_sc_frame =
                         (usb->frame_number + 1) & 0x7f;
                 else
                     pipe->split_sc_frame =
                         (usb->frame_number + 2) & 0x7f;
             } else {
                 pipe->split_sc_frame = -1;
             }
 
             usbc_hcsplt.s.spltena = 1;
             usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
             usbc_hcsplt.s.prtaddr = pipe->hub_port;
             usbc_hcsplt.s.compsplt = (transaction->stage ==
                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
 
             /*
              * SPLIT transactions can only ever transmit one data
              * packet so limit the transfer size to the max packet
              * size
              */
             if (bytes_to_transfer > pipe->max_packet)
                 bytes_to_transfer = pipe->max_packet;
 
             /*
              * ISOCHRONOUS OUT splits are unique in that they limit
              * data transfers to 188 byte chunks representing the
              * begin/middle/end of the data or all
              */
             if (!usbc_hcsplt.s.compsplt &&
                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
                 (pipe->transfer_type ==
                  CVMX_USB_TRANSFER_ISOCHRONOUS)) {
                 /*
                  * Clear the split complete frame number as
                  * there isn't going to be a split complete
                  */
                 pipe->split_sc_frame = -1;
                 /*
                  * See if we've started this transfer and sent
                  * data
                  */
                 if (transaction->actual_bytes == 0) {
                     /*
                      * Nothing sent yet, this is either a
                      * begin or the entire payload
                      */
                     if (bytes_to_transfer <= 188)
                         /* Entire payload in one go */
                         usbc_hcsplt.s.xactpos = 3;
                     else
                         /* First part of payload */
                         usbc_hcsplt.s.xactpos = 2;
                 } else {
                     /*
                      * Continuing the previous data, we must
                      * either be in the middle or at the end
                      */
                     if (bytes_to_transfer <= 188)
                         /* End of payload */
                         usbc_hcsplt.s.xactpos = 1;
                     else
                         /* Middle of payload */
                         usbc_hcsplt.s.xactpos = 0;
                 }
                 /*
                  * Again, the transfer size is limited to 188
                  * bytes
                  */
                 if (bytes_to_transfer > 188)
                     bytes_to_transfer = 188;
             }
         }
 
         /*
          * Make sure the transfer never exceeds the byte limit of the
          * hardware. Further bytes will be sent as continued
          * transactions
          */
         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
             /*
              * Round MAX_TRANSFER_BYTES to a multiple of out packet
              * size
              */
             bytes_to_transfer = MAX_TRANSFER_BYTES /
                 pipe->max_packet;
             bytes_to_transfer *= pipe->max_packet;
         }
 
         /*
          * Calculate the number of packets to transfer. If the length is
          * zero we still need to transfer one packet
          */
         packets_to_transfer =
             DIV_ROUND_UP(bytes_to_transfer, pipe->max_packet);
         if (packets_to_transfer == 0) {
             packets_to_transfer = 1;
         } else if ((packets_to_transfer > 1) &&
                (usb->init_flags &
                 CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
             /*
              * Limit to one packet when not using DMA. Channels must
              * be restarted between every packet for IN
              * transactions, so there is no reason to do multiple
              * packets in a row
              */
             packets_to_transfer = 1;
             bytes_to_transfer = packets_to_transfer *
                 pipe->max_packet;
         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
             /*
              * Limit the number of packet and data transferred to
              * what the hardware can handle
              */
             packets_to_transfer = MAX_TRANSFER_PACKETS;
             bytes_to_transfer = packets_to_transfer *
                 pipe->max_packet;
         }
 
         usbc_hctsiz.s.xfersize = bytes_to_transfer;
         usbc_hctsiz.s.pktcnt = packets_to_transfer;
 
         /* Update the DATA0/DATA1 toggle */
         usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
         /*
          * High speed pipes may need a hardware ping before they start
          */
         if (pipe->flags & CVMX_USB_PIPE_FLAGS_NEED_PING)
             usbc_hctsiz.s.dopng = 1;
 
         cvmx_usb_write_csr32(usb,
                      CVMX_USBCX_HCSPLTX(channel, usb->index),
                      usbc_hcsplt.u32);
         cvmx_usb_write_csr32(usb,
                      CVMX_USBCX_HCTSIZX(channel, usb->index),
                      usbc_hctsiz.u32);
     }
 
     /* Setup the Host Channel Characteristics Register */
     {
         union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
 
         /*
          * Set the startframe odd/even properly. This is only used for
          * periodic
          */
         usbc_hcchar.s.oddfrm = usb->frame_number & 1;
 
         /*
          * Set the number of back to back packets allowed by this
          * endpoint. Split transactions interpret "ec" as the number of
          * immediate retries of failure. These retries happen too
          * quickly, so we disable these entirely for splits
          */
         if (cvmx_usb_pipe_needs_split(usb, pipe))
             usbc_hcchar.s.ec = 1;
         else if (pipe->multi_count < 1)
             usbc_hcchar.s.ec = 1;
         else if (pipe->multi_count > 3)
             usbc_hcchar.s.ec = 3;
         else
             usbc_hcchar.s.ec = pipe->multi_count;
 
         /* Set the rest of the endpoint specific settings */
         usbc_hcchar.s.devaddr = pipe->device_addr;
         usbc_hcchar.s.eptype = transaction->type;
         usbc_hcchar.s.lspddev =
             (pipe->device_speed == CVMX_USB_SPEED_LOW);
         usbc_hcchar.s.epdir = pipe->transfer_dir;
         usbc_hcchar.s.epnum = pipe->endpoint_num;
         usbc_hcchar.s.mps = pipe->max_packet;
         cvmx_usb_write_csr32(usb,
                      CVMX_USBCX_HCCHARX(channel, usb->index),
                      usbc_hcchar.u32);
     }
 
     /* Do transaction type specific fixups as needed */
     switch (transaction->type) {
     case CVMX_USB_TRANSFER_CONTROL:
         cvmx_usb_start_channel_control(usb, channel, pipe);
         break;
     case CVMX_USB_TRANSFER_BULK:
     case CVMX_USB_TRANSFER_INTERRUPT:
         break;
     case CVMX_USB_TRANSFER_ISOCHRONOUS:
         if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
             /*
              * ISO transactions require different PIDs depending on
              * direction and how many packets are needed
              */
             if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
                 if (pipe->multi_count < 2) /* Need DATA0 */
                     USB_SET_FIELD32(
                         CVMX_USBCX_HCTSIZX(channel,
                                    usb->index),
                         cvmx_usbcx_hctsizx, pid, 0);
                 else /* Need MDATA */
                     USB_SET_FIELD32(
                         CVMX_USBCX_HCTSIZX(channel,
                                    usb->index),
                         cvmx_usbcx_hctsizx, pid, 3);
             }
         }
         break;
     }
     {
         union cvmx_usbcx_hctsizx usbc_hctsiz = { .u32 =
             cvmx_usb_read_csr32(usb,
                         CVMX_USBCX_HCTSIZX(channel,
                                    usb->index))
         };
         transaction->xfersize = usbc_hctsiz.s.xfersize;
         transaction->pktcnt = usbc_hctsiz.s.pktcnt;
     }
     /* Remember when we start a split transaction */
     if (cvmx_usb_pipe_needs_split(usb, pipe))
         usb->active_split = transaction;
     USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
             cvmx_usbcx_hccharx, chena, 1);
     if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
         cvmx_usb_fill_tx_fifo(usb, channel);
 }
 
 /**
  * Find a pipe that is ready to be scheduled to hardware.
  * @usb:     USB device state populated by cvmx_usb_initialize().
  * @xfer_type:   Transfer type
  *
  * Returns: Pipe or NULL if none are ready
  */
 static struct cvmx_usb_pipe *cvmx_usb_find_ready_pipe(struct octeon_hcd *usb,
                               enum cvmx_usb_transfer xfer_type)
 {
     struct list_head *list = usb->active_pipes + xfer_type;
     u64 current_frame = usb->frame_number;
     struct cvmx_usb_pipe *pipe;
 
     list_for_each_entry(pipe, list, node) {
         struct cvmx_usb_transaction *t =
             list_first_entry(&pipe->transactions, typeof(*t),
                      node);
         if (!(pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
             (pipe->next_tx_frame <= current_frame) &&
             ((pipe->split_sc_frame == -1) ||
              ((((int)current_frame - pipe->split_sc_frame) & 0x7f) <
               0x40)) &&
             (!usb->active_split || (usb->active_split == t))) {
             prefetch(t);
             return pipe;
         }
     }
     return NULL;
 }
 
 static struct cvmx_usb_pipe *cvmx_usb_next_pipe(struct octeon_hcd *usb,
                         int is_sof)
 {
     struct cvmx_usb_pipe *pipe;
 
     /* Find a pipe needing service. */
     if (is_sof) {
         /*
          * Only process periodic pipes on SOF interrupts. This way we
          * are sure that the periodic data is sent in the beginning of
          * the frame.
          */
         pipe = cvmx_usb_find_ready_pipe(usb,
                         CVMX_USB_TRANSFER_ISOCHRONOUS);
         if (pipe)
             return pipe;
         pipe = cvmx_usb_find_ready_pipe(usb,
                         CVMX_USB_TRANSFER_INTERRUPT);
         if (pipe)
             return pipe;
     }
     pipe = cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_CONTROL);
     if (pipe)
         return pipe;
     return cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_BULK);
 }
 
 /**
  * Called whenever a pipe might need to be scheduled to the
  * hardware.
  *
  * @usb:     USB device state populated by cvmx_usb_initialize().
  * @is_sof:  True if this schedule was called on a SOF interrupt.
  */
 static void cvmx_usb_schedule(struct octeon_hcd *usb, int is_sof)
 {
     int channel;
     struct cvmx_usb_pipe *pipe;
     int need_sof;
     enum cvmx_usb_transfer ttype;
 
     if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
         /*
          * Without DMA we need to be careful to not schedule something
          * at the end of a frame and cause an overrun.
          */
         union cvmx_usbcx_hfnum hfnum = {
             .u32 = cvmx_usb_read_csr32(usb,
                         CVMX_USBCX_HFNUM(usb->index))
         };
 
         union cvmx_usbcx_hfir hfir = {
             .u32 = cvmx_usb_read_csr32(usb,
                         CVMX_USBCX_HFIR(usb->index))
         };
 
         if (hfnum.s.frrem < hfir.s.frint / 4)
             goto done;
     }
 
     while (usb->idle_hardware_channels) {
         /* Find an idle channel */
         channel = __fls(usb->idle_hardware_channels);
         if (unlikely(channel > 7))
             break;
 
         pipe = cvmx_usb_next_pipe(usb, is_sof);
         if (!pipe)
             break;
 
         cvmx_usb_start_channel(usb, channel, pipe);
     }
 
 done:
     /*
      * Only enable SOF interrupts when we have transactions pending in the
      * future that might need to be scheduled
      */
     need_sof = 0;
     for (ttype = CVMX_USB_TRANSFER_CONTROL;
          ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
         list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
             if (pipe->next_tx_frame > usb->frame_number) {
                 need_sof = 1;
                 break;
             }
         }
     }
     USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
             cvmx_usbcx_gintmsk, sofmsk, need_sof);
 }
 
 static void octeon_usb_urb_complete_callback(struct octeon_hcd *usb,
                          enum cvmx_usb_status status,
                          struct cvmx_usb_pipe *pipe,
                          struct cvmx_usb_transaction
                         *transaction,
                          int bytes_transferred,
                          struct urb *urb)
 {
     struct usb_hcd *hcd = octeon_to_hcd(usb);
     struct device *dev = hcd->self.controller;
 
     if (likely(status == CVMX_USB_STATUS_OK))
         urb->actual_length = bytes_transferred;
     else
         urb->actual_length = 0;
 
     urb->hcpriv = NULL;
 
     /* For Isochronous transactions we need to update the URB packet status
      * list from data in our private copy
      */
     if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
         int i;
         /*
          * The pointer to the private list is stored in the setup_packet
          * field.
          */
         struct cvmx_usb_iso_packet *iso_packet =
             (struct cvmx_usb_iso_packet *)urb->setup_packet;
         /* Recalculate the transfer size by adding up each packet */
         urb->actual_length = 0;
         for (i = 0; i < urb->number_of_packets; i++) {
             if (iso_packet[i].status == CVMX_USB_STATUS_OK) {
                 urb->iso_frame_desc[i].status = 0;
                 urb->iso_frame_desc[i].actual_length =
                     iso_packet[i].length;
                 urb->actual_length +=
                     urb->iso_frame_desc[i].actual_length;
             } else {
                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
                     i, urb->number_of_packets,
                     iso_packet[i].status, pipe,
                     transaction, iso_packet[i].length);
                 urb->iso_frame_desc[i].status = -EREMOTEIO;
             }
         }
         /* Free the private list now that we don't need it anymore */
         kfree(iso_packet);
         urb->setup_packet = NULL;
     }
 
     switch (status) {
     case CVMX_USB_STATUS_OK:
         urb->status = 0;
         break;
     case CVMX_USB_STATUS_CANCEL:
         if (urb->status == 0)
             urb->status = -ENOENT;
         break;
     case CVMX_USB_STATUS_STALL:
         dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
             pipe, transaction, bytes_transferred);
         urb->status = -EPIPE;
         break;
     case CVMX_USB_STATUS_BABBLEERR:
         dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
             pipe, transaction, bytes_transferred);
         urb->status = -EPIPE;
         break;
     case CVMX_USB_STATUS_SHORT:
         dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
             pipe, transaction, bytes_transferred);
         urb->status = -EREMOTEIO;
         break;
     case CVMX_USB_STATUS_ERROR:
     case CVMX_USB_STATUS_XACTERR:
     case CVMX_USB_STATUS_DATATGLERR:
     case CVMX_USB_STATUS_FRAMEERR:
         dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
             status, pipe, transaction, bytes_transferred);
         urb->status = -EPROTO;
         break;
     }
     usb_hcd_unlink_urb_from_ep(octeon_to_hcd(usb), urb);
     spin_unlock(&usb->lock);
     usb_hcd_giveback_urb(octeon_to_hcd(usb), urb, urb->status);
     spin_lock(&usb->lock);
 }
 
 /**
  * Signal the completion of a transaction and free it. The
  * transaction will be removed from the pipe transaction list.
  *
  * @usb:     USB device state populated by cvmx_usb_initialize().
  * @pipe:    Pipe the transaction is on
  * @transaction:
  *       Transaction that completed
  * @complete_code:
  *       Completion code
  */
 static void cvmx_usb_complete(struct octeon_hcd *usb,
                   struct cvmx_usb_pipe *pipe,
                   struct cvmx_usb_transaction *transaction,
                   enum cvmx_usb_status complete_code)
 {
     /* If this was a split then clear our split in progress marker */
     if (usb->active_split == transaction)
         usb->active_split = NULL;
 
     /*
      * Isochronous transactions need extra processing as they might not be
      * done after a single data transfer
      */
     if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
         /* Update the number of bytes transferred in this ISO packet */
         transaction->iso_packets[0].length = transaction->actual_bytes;
         transaction->iso_packets[0].status = complete_code;
 
         /*
          * If there are more ISOs pending and we succeeded, schedule the
          * next one
          */
         if ((transaction->iso_number_packets > 1) &&
             (complete_code == CVMX_USB_STATUS_OK)) {
             /* No bytes transferred for this packet as of yet */
             transaction->actual_bytes = 0;
             /* One less ISO waiting to transfer */
             transaction->iso_number_packets--;
             /* Increment to the next location in our packet array */
             transaction->iso_packets++;
             transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
             return;
         }
     }
 
     /* Remove the transaction from the pipe list */
     list_del(&transaction->node);
     if (list_empty(&pipe->transactions))
         list_move_tail(&pipe->node, &usb->idle_pipes);
     octeon_usb_urb_complete_callback(usb, complete_code, pipe,
                      transaction,
                      transaction->actual_bytes,
                      transaction->urb);
     kfree(transaction);
 }
 
 /**
  * Submit a usb transaction to a pipe. Called for all types
  * of transactions.
  *
  * @usb:
  * @pipe:       Which pipe to submit to.
  * @type:       Transaction type
  * @buffer:     User buffer for the transaction
  * @buffer_length:
  *          User buffer's length in bytes
  * @control_header:
  *          For control transactions, the 8 byte standard header
  * @iso_start_frame:
  *          For ISO transactions, the start frame
  * @iso_number_packets:
  *          For ISO, the number of packet in the transaction.
  * @iso_packets:
  *          A description of each ISO packet
  * @urb:        URB for the callback
  *
  * Returns: Transaction or NULL on failure.
  */
 static struct cvmx_usb_transaction *cvmx_usb_submit_transaction(
                 struct octeon_hcd *usb,
                 struct cvmx_usb_pipe *pipe,
                 enum cvmx_usb_transfer type,
                 u64 buffer,
                 int buffer_length,
                 u64 control_header,
                 int iso_start_frame,
                 int iso_number_packets,
                 struct cvmx_usb_iso_packet *iso_packets,
                 struct urb *urb)
 {
     struct cvmx_usb_transaction *transaction;
 
     if (unlikely(pipe->transfer_type != type))
         return NULL;
 
     transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
     if (unlikely(!transaction))
         return NULL;
 
     transaction->type = type;
     transaction->buffer = buffer;
     transaction->buffer_length = buffer_length;
     transaction->control_header = control_header;
     /* FIXME: This is not used, implement it. */
     transaction->iso_start_frame = iso_start_frame;
     transaction->iso_number_packets = iso_number_packets;
     transaction->iso_packets = iso_packets;
     transaction->urb = urb;
     if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
         transaction->stage = CVMX_USB_STAGE_SETUP;
     else
         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
 
     if (!list_empty(&pipe->transactions)) {
         list_add_tail(&transaction->node, &pipe->transactions);
     } else {
         list_add_tail(&transaction->node, &pipe->transactions);
         list_move_tail(&pipe->node,
                    &usb->active_pipes[pipe->transfer_type]);
 
         /*
          * We may need to schedule the pipe if this was the head of the
          * pipe.
          */
         cvmx_usb_schedule(usb, 0);
     }
 
     return transaction;
 }
 
 /**
  * Call to submit a USB Bulk transfer to a pipe.
  *
  * @usb:        USB device state populated by cvmx_usb_initialize().
  * @pipe:       Handle to the pipe for the transfer.
  * @urb:        URB.
  *
  * Returns: A submitted transaction or NULL on failure.
  */
 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
                         struct octeon_hcd *usb,
                         struct cvmx_usb_pipe *pipe,
                         struct urb *urb)
 {
     return cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
                        urb->transfer_dma,
                        urb->transfer_buffer_length,
                        0, /* control_header */
                        0, /* iso_start_frame */
                        0, /* iso_number_packets */
                        NULL, /* iso_packets */
                        urb);
 }
 
 /**
  * Call to submit a USB Interrupt transfer to a pipe.
  *
  * @usb:        USB device state populated by cvmx_usb_initialize().
  * @pipe:       Handle to the pipe for the transfer.
  * @urb:        URB returned when the callback is called.
  *
  * Returns: A submitted transaction or NULL on failure.
  */
 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
                         struct octeon_hcd *usb,
                         struct cvmx_usb_pipe *pipe,
                         struct urb *urb)
 {
     return cvmx_usb_submit_transaction(usb, pipe,
                        CVMX_USB_TRANSFER_INTERRUPT,
                        urb->transfer_dma,
                        urb->transfer_buffer_length,
                        0, /* control_header */
                        0, /* iso_start_frame */
                        0, /* iso_number_packets */
                        NULL, /* iso_packets */
                        urb);
 }
 
 /**
  * Call to submit a USB Control transfer to a pipe.
  *
  * @usb:        USB device state populated by cvmx_usb_initialize().
  * @pipe:       Handle to the pipe for the transfer.
  * @urb:        URB.
  *
  * Returns: A submitted transaction or NULL on failure.
  */
 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
                         struct octeon_hcd *usb,
                         struct cvmx_usb_pipe *pipe,
                         struct urb *urb)
 {
     int buffer_length = urb->transfer_buffer_length;
     u64 control_header = urb->setup_dma;
     struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);
 
     if ((header->bRequestType & USB_DIR_IN) == 0)
         buffer_length = le16_to_cpu(header->wLength);
 
     return cvmx_usb_submit_transaction(usb, pipe,
                        CVMX_USB_TRANSFER_CONTROL,
                        urb->transfer_dma, buffer_length,
                        control_header,
                        0, /* iso_start_frame */
                        0, /* iso_number_packets */
                        NULL, /* iso_packets */
                        urb);
 }
 
 /**
  * Call to submit a USB Isochronous transfer to a pipe.
  *
  * @usb:        USB device state populated by cvmx_usb_initialize().
  * @pipe:       Handle to the pipe for the transfer.
  * @urb:        URB returned when the callback is called.
  *
  * Returns: A submitted transaction or NULL on failure.
  */
 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
                         struct octeon_hcd *usb,
                         struct cvmx_usb_pipe *pipe,
                         struct urb *urb)
 {
     struct cvmx_usb_iso_packet *packets;
 
     packets = (struct cvmx_usb_iso_packet *)urb->setup_packet;
     return cvmx_usb_submit_transaction(usb, pipe,
                        CVMX_USB_TRANSFER_ISOCHRONOUS,
                        urb->transfer_dma,
                        urb->transfer_buffer_length,
                        0, /* control_header */
                        urb->start_frame,
                        urb->number_of_packets,
                        packets, urb);
 }
 
 /**
  * Cancel one outstanding request in a pipe. Canceling a request
  * can fail if the transaction has already completed before cancel
  * is called. Even after a successful cancel call, it may take
  * a frame or two for the cvmx_usb_poll() function to call the
  * associated callback.
  *
  * @usb:     USB device state populated by cvmx_usb_initialize().
  * @pipe:    Pipe to cancel requests in.
  * @transaction: Transaction to cancel, returned by the submit function.
  *
  * Returns: 0 or a negative error code.
  */
 static int cvmx_usb_cancel(struct octeon_hcd *usb,
                struct cvmx_usb_pipe *pipe,
                struct cvmx_usb_transaction *transaction)
 {
     /*
      * If the transaction is the HEAD of the queue and scheduled. We need to
      * treat it special
      */
     if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
         transaction && (pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
         union cvmx_usbcx_hccharx usbc_hcchar;
 
         usb->pipe_for_channel[pipe->channel] = NULL;
         pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
 
         CVMX_SYNCW;
 
         usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
                               CVMX_USBCX_HCCHARX(pipe->channel,
                                      usb->index));
         /*
          * If the channel isn't enabled then the transaction already
          * completed.
          */
         if (usbc_hcchar.s.chena) {
             usbc_hcchar.s.chdis = 1;
             cvmx_usb_write_csr32(usb,
                          CVMX_USBCX_HCCHARX(pipe->channel,
                                 usb->index),
                          usbc_hcchar.u32);
         }
     }
     cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_CANCEL);
     return 0;
 }
 
 /**
  * Cancel all outstanding requests in a pipe. Logically all this
  * does is call cvmx_usb_cancel() in a loop.
  *
  * @usb:     USB device state populated by cvmx_usb_initialize().
  * @pipe:    Pipe to cancel requests in.
  *
  * Returns: 0 or a negative error code.
  */
 static int cvmx_usb_cancel_all(struct octeon_hcd *usb,
                    struct cvmx_usb_pipe *pipe)
 {
     struct cvmx_usb_transaction *transaction, *next;
 
     /* Simply loop through and attempt to cancel each transaction */
     list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
         int result = cvmx_usb_cancel(usb, pipe, transaction);
 
         if (unlikely(result != 0))
             return result;
     }
     return 0;
 }
 
 /**
  * Close a pipe created with cvmx_usb_open_pipe().
  *
  * @usb:     USB device state populated by cvmx_usb_initialize().
  * @pipe:    Pipe to close.
  *
  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
  *      outstanding transfers.
  */
 static int cvmx_usb_close_pipe(struct octeon_hcd *usb,
                    struct cvmx_usb_pipe *pipe)
 {
     /* Fail if the pipe has pending transactions */
     if (!list_empty(&pipe->transactions))
         return -EBUSY;
 
     list_del(&pipe->node);
     kfree(pipe);
 
     return 0;
 }
 
 /**
  * Get the current USB protocol level frame number. The frame
  * number is always in the range of 0-0x7ff.
  *
  * @usb: USB device state populated by cvmx_usb_initialize().
  *
  * Returns: USB frame number
  */
 static int cvmx_usb_get_frame_number(struct octeon_hcd *usb)
 {
     union cvmx_usbcx_hfnum usbc_hfnum;
 
     usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
 
     return usbc_hfnum.s.frnum;
 }
 
 static void cvmx_usb_transfer_control(struct octeon_hcd *usb,
                       struct cvmx_usb_pipe *pipe,
                       struct cvmx_usb_transaction *transaction,
                       union cvmx_usbcx_hccharx usbc_hcchar,
                       int buffer_space_left,
                       int bytes_in_last_packet)
 {
     switch (transaction->stage) {
     case CVMX_USB_STAGE_NON_CONTROL:
     case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
         /* This should be impossible */
         cvmx_usb_complete(usb, pipe, transaction,
                   CVMX_USB_STATUS_ERROR);
         break;
     case CVMX_USB_STAGE_SETUP:
         pipe->pid_toggle = 1;
         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
             transaction->stage =
                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
         } else {
             struct usb_ctrlrequest *header =
                 cvmx_phys_to_ptr(transaction->control_header);
             if (header->wLength)
                 transaction->stage = CVMX_USB_STAGE_DATA;
             else
                 transaction->stage = CVMX_USB_STAGE_STATUS;
         }
         break;
     case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
         {
             struct usb_ctrlrequest *header =
                 cvmx_phys_to_ptr(transaction->control_header);
             if (header->wLength)
                 transaction->stage = CVMX_USB_STAGE_DATA;
             else
                 transaction->stage = CVMX_USB_STAGE_STATUS;
         }
         break;
     case CVMX_USB_STAGE_DATA:
         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
             transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
             /*
              * For setup OUT data that are splits,
              * the hardware doesn't appear to count
              * transferred data. Here we manually
              * update the data transferred
              */
             if (!usbc_hcchar.s.epdir) {
                 if (buffer_space_left < pipe->max_packet)
                     transaction->actual_bytes +=
                         buffer_space_left;
                 else
                     transaction->actual_bytes +=
                         pipe->max_packet;
             }
         } else if ((buffer_space_left == 0) ||
                (bytes_in_last_packet < pipe->max_packet)) {
             pipe->pid_toggle = 1;
             transaction->stage = CVMX_USB_STAGE_STATUS;
         }
         break;
     case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
         if ((buffer_space_left == 0) ||
             (bytes_in_last_packet < pipe->max_packet)) {
             pipe->pid_toggle = 1;
             transaction->stage = CVMX_USB_STAGE_STATUS;
         } else {
             transaction->stage = CVMX_USB_STAGE_DATA;
         }
         break;
     case CVMX_USB_STAGE_STATUS:
         if (cvmx_usb_pipe_needs_split(usb, pipe))
             transaction->stage =
                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
         else
             cvmx_usb_complete(usb, pipe, transaction,
                       CVMX_USB_STATUS_OK);
         break;
     case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
         cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
         break;
     }
 }
 
 static void cvmx_usb_transfer_bulk(struct octeon_hcd *usb,
                    struct cvmx_usb_pipe *pipe,
                    struct cvmx_usb_transaction *transaction,
                    union cvmx_usbcx_hcintx usbc_hcint,
                    int buffer_space_left,
                    int bytes_in_last_packet)
 {
     /*
      * The only time a bulk transfer isn't complete when it finishes with
      * an ACK is during a split transaction. For splits we need to continue
      * the transfer if more data is needed.
      */
     if (cvmx_usb_pipe_needs_split(usb, pipe)) {
         if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
             transaction->stage =
                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
         else if (buffer_space_left &&
              (bytes_in_last_packet == pipe->max_packet))
             transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
         else
             cvmx_usb_complete(usb, pipe, transaction,
                       CVMX_USB_STATUS_OK);
     } else {
         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
             (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
             (usbc_hcint.s.nak))
             pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
         if (!buffer_space_left ||
             (bytes_in_last_packet < pipe->max_packet))
             cvmx_usb_complete(usb, pipe, transaction,
                       CVMX_USB_STATUS_OK);
     }
 }
 
 static void cvmx_usb_transfer_intr(struct octeon_hcd *usb,
                    struct cvmx_usb_pipe *pipe,
                    struct cvmx_usb_transaction *transaction,
                    int buffer_space_left,
                    int bytes_in_last_packet)
 {
     if (cvmx_usb_pipe_needs_split(usb, pipe)) {
         if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL) {
             transaction->stage =
                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
         } else if (buffer_space_left &&
                (bytes_in_last_packet == pipe->max_packet)) {
             transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
         } else {
             pipe->next_tx_frame += pipe->interval;
             cvmx_usb_complete(usb, pipe, transaction,
                       CVMX_USB_STATUS_OK);
         }
     } else if (!buffer_space_left ||
            (bytes_in_last_packet < pipe->max_packet)) {
         pipe->next_tx_frame += pipe->interval;
         cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
     }
 }
 
 static void cvmx_usb_transfer_isoc(struct octeon_hcd *usb,
                    struct cvmx_usb_pipe *pipe,
                    struct cvmx_usb_transaction *transaction,
                    int buffer_space_left,
                    int bytes_in_last_packet,
                    int bytes_this_transfer)
 {
     if (cvmx_usb_pipe_needs_split(usb, pipe)) {
         /*
          * ISOCHRONOUS OUT splits don't require a complete split stage.
          * Instead they use a sequence of begin OUT splits to transfer
          * the data 188 bytes at a time. Once the transfer is complete,
          * the pipe sleeps until the next schedule interval.
          */
         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
             /*
              * If no space left or this wasn't a max size packet
              * then this transfer is complete. Otherwise start it
              * again to send the next 188 bytes
              */
             if (!buffer_space_left || (bytes_this_transfer < 188)) {
                 pipe->next_tx_frame += pipe->interval;
                 cvmx_usb_complete(usb, pipe, transaction,
                           CVMX_USB_STATUS_OK);
             }
             return;
         }
         if (transaction->stage ==
             CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
             /*
              * We are in the incoming data phase. Keep getting data
              * until we run out of space or get a small packet
              */
             if ((buffer_space_left == 0) ||
                 (bytes_in_last_packet < pipe->max_packet)) {
                 pipe->next_tx_frame += pipe->interval;
                 cvmx_usb_complete(usb, pipe, transaction,
                           CVMX_USB_STATUS_OK);
             }
         } else {
             transaction->stage =
                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
         }
     } else {
         pipe->next_tx_frame += pipe->interval;
         cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
     }
 }
 
 /**
  * Poll a channel for status
  *
  * @usb:     USB device
  * @channel: Channel to poll
  *
  * Returns: Zero on success
  */
 static int cvmx_usb_poll_channel(struct octeon_hcd *usb, int channel)
 {
     struct usb_hcd *hcd = octeon_to_hcd(usb);
     struct device *dev = hcd->self.controller;
     union cvmx_usbcx_hcintx usbc_hcint;
     union cvmx_usbcx_hctsizx usbc_hctsiz;
     union cvmx_usbcx_hccharx usbc_hcchar;
     struct cvmx_usb_pipe *pipe;
     struct cvmx_usb_transaction *transaction;
     int bytes_this_transfer;
     int bytes_in_last_packet;
     int packets_processed;
     int buffer_space_left;
 
     /* Read the interrupt status bits for the channel */
     usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
                          CVMX_USBCX_HCINTX(channel, usb->index));
 
     if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
         usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
                               CVMX_USBCX_HCCHARX(channel,
                                      usb->index));
 
         if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
             /*
              * There seems to be a bug in CN31XX which can cause
              * interrupt IN transfers to get stuck until we do a
              * write of HCCHARX without changing things
              */
             cvmx_usb_write_csr32(usb,
                          CVMX_USBCX_HCCHARX(channel,
                                 usb->index),
                          usbc_hcchar.u32);
             return 0;
         }
 
         /*
          * In non DMA mode the channels don't halt themselves. We need
          * to manually disable channels that are left running
          */
         if (!usbc_hcint.s.chhltd) {
             if (usbc_hcchar.s.chena) {
                 union cvmx_usbcx_hcintmskx hcintmsk;
                 /* Disable all interrupts except CHHLTD */
                 hcintmsk.u32 = 0;
                 hcintmsk.s.chhltdmsk = 1;
                 cvmx_usb_write_csr32(usb,
                              CVMX_USBCX_HCINTMSKX(channel, usb->index),
                              hcintmsk.u32);
                 usbc_hcchar.s.chdis = 1;
                 cvmx_usb_write_csr32(usb,
                              CVMX_USBCX_HCCHARX(channel, usb->index),
                              usbc_hcchar.u32);
                 return 0;
             } else if (usbc_hcint.s.xfercompl) {
                 /*
                  * Successful IN/OUT with transfer complete.
                  * Channel halt isn't needed.
                  */
             } else {
                 dev_err(dev, "USB%d: Channel %d interrupt without halt\n",
                     usb->index, channel);
                 return 0;
             }
         }
     } else {
         /*
          * There is are no interrupts that we need to process when the
          * channel is still running
          */
         if (!usbc_hcint.s.chhltd)
             return 0;
     }
 
     /* Disable the channel interrupts now that it is done */
     cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
     usb->idle_hardware_channels |= (1 << channel);
 
     /* Make sure this channel is tied to a valid pipe */
     pipe = usb->pipe_for_channel[channel];
     prefetch(pipe);
     if (!pipe)
         return 0;
     transaction = list_first_entry(&pipe->transactions,
                        typeof(*transaction),
                        node);
     prefetch(transaction);
 
     /*
      * Disconnect this pipe from the HW channel. Later the schedule
      * function will figure out which pipe needs to go
      */
     usb->pipe_for_channel[channel] = NULL;
     pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
 
     /*
      * Read the channel config info so we can figure out how much data
      * transferred
      */
     usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
                           CVMX_USBCX_HCCHARX(channel, usb->index));
     usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
                           CVMX_USBCX_HCTSIZX(channel, usb->index));
 
     /*
      * Calculating the number of bytes successfully transferred is dependent
      * on the transfer direction
      */
     packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
     if (usbc_hcchar.s.epdir) {
         /*
          * IN transactions are easy. For every byte received the
          * hardware decrements xfersize. All we need to do is subtract
          * the current value of xfersize from its starting value and we
          * know how many bytes were written to the buffer
          */
         bytes_this_transfer = transaction->xfersize -
             usbc_hctsiz.s.xfersize;
     } else {
         /*
          * OUT transaction don't decrement xfersize. Instead pktcnt is
          * decremented on every successful packet send. The hardware
          * does this when it receives an ACK, or NYET. If it doesn't
          * receive one of these responses pktcnt doesn't change
          */
         bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
         /*
          * The last packet may not be a full transfer if we didn't have
          * enough data
          */
         if (bytes_this_transfer > transaction->xfersize)
             bytes_this_transfer = transaction->xfersize;
     }
     /* Figure out how many bytes were in the last packet of the transfer */
     if (packets_processed)
         bytes_in_last_packet = bytes_this_transfer -
             (packets_processed - 1) * usbc_hcchar.s.mps;
     else
         bytes_in_last_packet = bytes_this_transfer;
 
     /*
      * As a special case, setup transactions output the setup header, not
      * the user's data. For this reason we don't count setup data as bytes
      * transferred
      */
     if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
         (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
         bytes_this_transfer = 0;
 
     /*
      * Add the bytes transferred to the running total. It is important that
      * bytes_this_transfer doesn't count any data that needs to be
      * retransmitted
      */
     transaction->actual_bytes += bytes_this_transfer;
     if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
         buffer_space_left = transaction->iso_packets[0].length -
             transaction->actual_bytes;
     else
         buffer_space_left = transaction->buffer_length -
             transaction->actual_bytes;
 
     /*
      * We need to remember the PID toggle state for the next transaction.
      * The hardware already updated it for the next transaction
      */
     pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
 
     /*
      * For high speed bulk out, assume the next transaction will need to do
      * a ping before proceeding. If this isn't true the ACK processing below
      * will clear this flag
      */
     if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
         (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
         (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
         pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
 
     if (WARN_ON_ONCE(bytes_this_transfer < 0)) {
         /*
          * In some rare cases the DMA engine seems to get stuck and
          * keeps substracting same byte count over and over again. In
          * such case we just need to fail every transaction.
          */
         cvmx_usb_complete(usb, pipe, transaction,
                   CVMX_USB_STATUS_ERROR);
         return 0;
     }
 
     if (usbc_hcint.s.stall) {
         /*
          * STALL as a response means this transaction cannot be
          * completed because the device can't process transactions. Tell
          * the user. Any data that was transferred will be counted on
          * the actual bytes transferred
          */
         pipe->pid_toggle = 0;
         cvmx_usb_complete(usb, pipe, transaction,
                   CVMX_USB_STATUS_STALL);
     } else if (usbc_hcint.s.xacterr) {
         /*
          * XactErr as a response means the device signaled
          * something wrong with the transfer. For example, PID
          * toggle errors cause these.
          */
         cvmx_usb_complete(usb, pipe, transaction,
                   CVMX_USB_STATUS_XACTERR);
     } else if (usbc_hcint.s.bblerr) {
         /* Babble Error (BblErr) */
         cvmx_usb_complete(usb, pipe, transaction,
                   CVMX_USB_STATUS_BABBLEERR);
     } else if (usbc_hcint.s.datatglerr) {
         /* Data toggle error */
         cvmx_usb_complete(usb, pipe, transaction,
                   CVMX_USB_STATUS_DATATGLERR);
     } else if (usbc_hcint.s.nyet) {
         /*
          * NYET as a response is only allowed in three cases: as a
          * response to a ping, as a response to a split transaction, and
          * as a response to a bulk out. The ping case is handled by
          * hardware, so we only have splits and bulk out
          */
         if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
             transaction->retries = 0;
             /*
              * If there is more data to go then we need to try
              * again. Otherwise this transaction is complete
              */
             if ((buffer_space_left == 0) ||
                 (bytes_in_last_packet < pipe->max_packet))
                 cvmx_usb_complete(usb, pipe,
                           transaction,
                           CVMX_USB_STATUS_OK);
         } else {
             /*
              * Split transactions retry the split complete 4 times
              * then rewind to the start split and do the entire
              * transactions again
              */
             transaction->retries++;
             if ((transaction->retries & 0x3) == 0) {
                 /*
                  * Rewind to the beginning of the transaction by
                  * anding off the split complete bit
                  */
                 transaction->stage &= ~1;
                 pipe->split_sc_frame = -1;
             }
         }
     } else if (usbc_hcint.s.ack) {
         transaction->retries = 0;
         /*
          * The ACK bit can only be checked after the other error bits.
          * This is because a multi packet transfer may succeed in a
          * number of packets and then get a different response on the
          * last packet. In this case both ACK and the last response bit
          * will be set. If none of the other response bits is set, then
          * the last packet must have been an ACK
          *
          * Since we got an ACK, we know we don't need to do a ping on
          * this pipe
          */
         pipe->flags &= ~CVMX_USB_PIPE_FLAGS_NEED_PING;
 
         switch (transaction->type) {
         case CVMX_USB_TRANSFER_CONTROL:
             cvmx_usb_transfer_control(usb, pipe, transaction,
                           usbc_hcchar,
                           buffer_space_left,
                           bytes_in_last_packet);
             break;
         case CVMX_USB_TRANSFER_BULK:
             cvmx_usb_transfer_bulk(usb, pipe, transaction,
                            usbc_hcint, buffer_space_left,
                            bytes_in_last_packet);
             break;
         case CVMX_USB_TRANSFER_INTERRUPT:
             cvmx_usb_transfer_intr(usb, pipe, transaction,
                            buffer_space_left,
                            bytes_in_last_packet);
             break;
         case CVMX_USB_TRANSFER_ISOCHRONOUS:
             cvmx_usb_transfer_isoc(usb, pipe, transaction,
                            buffer_space_left,
                            bytes_in_last_packet,
                            bytes_this_transfer);
             break;
         }
     } else if (usbc_hcint.s.nak) {
         /*
          * If this was a split then clear our split in progress marker.
          */
         if (usb->active_split == transaction)
             usb->active_split = NULL;
         /*
          * NAK as a response means the device couldn't accept the
          * transaction, but it should be retried in the future. Rewind
          * to the beginning of the transaction by anding off the split
          * complete bit. Retry in the next interval
          */
         transaction->retries = 0;
         transaction->stage &= ~1;
         pipe->next_tx_frame += pipe->interval;
         if (pipe->next_tx_frame < usb->frame_number)
             pipe->next_tx_frame = usb->frame_number +
                 pipe->interval -
                 (usb->frame_number - pipe->next_tx_frame) %
                 pipe->interval;
     } else {
         struct cvmx_usb_port_status port;
 
         port = cvmx_usb_get_status(usb);
         if (port.port_enabled) {
             /* We'll retry the exact same transaction again */
             transaction->retries++;
         } else {
             /*
              * We get channel halted interrupts with no result bits
              * sets when the cable is unplugged
              */
             cvmx_usb_complete(usb, pipe, transaction,
                       CVMX_USB_STATUS_ERROR);
         }
     }
     return 0;
 }
 
 static void octeon_usb_port_callback(struct octeon_hcd *usb)
 {
     spin_unlock(&usb->lock);
     usb_hcd_poll_rh_status(octeon_to_hcd(usb));
     spin_lock(&usb->lock);
 }
 
 /**
  * Poll the USB block for status and call all needed callback
  * handlers. This function is meant to be called in the interrupt
  * handler for the USB controller. It can also be called
  * periodically in a loop for non-interrupt based operation.
  *
  * @usb: USB device state populated by cvmx_usb_initialize().
  *
  * Returns: 0 or a negative error code.
  */
 static int cvmx_usb_poll(struct octeon_hcd *usb)
 {
     union cvmx_usbcx_hfnum usbc_hfnum;
     union cvmx_usbcx_gintsts usbc_gintsts;
 
     prefetch_range(usb, sizeof(*usb));
 
     /* Update the frame counter */
     usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
     if ((usb->frame_number & 0x3fff) > usbc_hfnum.s.frnum)
         usb->frame_number += 0x4000;
     usb->frame_number &= ~0x3fffull;
     usb->frame_number |= usbc_hfnum.s.frnum;
 
     /* Read the pending interrupts */
     usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
                            CVMX_USBCX_GINTSTS(usb->index));
 
     /* Clear the interrupts now that we know about them */
     cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
                  usbc_gintsts.u32);
 
     if (usbc_gintsts.s.rxflvl) {
         /*
          * RxFIFO Non-Empty (RxFLvl)
          * Indicates that there is at least one packet pending to be
          * read from the RxFIFO.
          *
          * In DMA mode this is handled by hardware
          */
         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
             cvmx_usb_poll_rx_fifo(usb);
     }
     if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
         /* Fill the Tx FIFOs when not in DMA mode */
         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
             cvmx_usb_poll_tx_fifo(usb);
     }
     if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
         union cvmx_usbcx_hprt usbc_hprt;
         /*
          * Disconnect Detected Interrupt (DisconnInt)
          * Asserted when a device disconnect is detected.
          *
          * Host Port Interrupt (PrtInt)
          * The core sets this bit to indicate a change in port status of
          * one of the O2P USB core ports in Host mode. The application
          * must read the Host Port Control and Status (HPRT) register to
          * determine the exact event that caused this interrupt. The
          * application must clear the appropriate status bit in the Host
          * Port Control and Status register to clear this bit.
          *
          * Call the user's port callback
          */
         octeon_usb_port_callback(usb);
         /* Clear the port change bits */
         usbc_hprt.u32 =
             cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
         usbc_hprt.s.prtena = 0;
         cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
                      usbc_hprt.u32);
     }
     if (usbc_gintsts.s.hchint) {
         /*
          * Host Channels Interrupt (HChInt)
          * The core sets this bit to indicate that an interrupt is
          * pending on one of the channels of the core (in Host mode).
          * The application must read the Host All Channels Interrupt
          * (HAINT) register to determine the exact number of the channel
          * on which the interrupt occurred, and then read the
          * corresponding Host Channel-n Interrupt (HCINTn) register to
          * determine the exact cause of the interrupt. The application
          * must clear the appropriate status bit in the HCINTn register
          * to clear this bit.
          */
         union cvmx_usbcx_haint usbc_haint;
 
         usbc_haint.u32 = cvmx_usb_read_csr32(usb,
                              CVMX_USBCX_HAINT(usb->index));
         while (usbc_haint.u32) {
             int channel;
 
             channel = __fls(usbc_haint.u32);
             cvmx_usb_poll_channel(usb, channel);
             usbc_haint.u32 ^= 1 << channel;
         }
     }
 
     cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
 
     return 0;
 }
 
 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
 {
     return (struct octeon_hcd *)(hcd->hcd_priv);
 }
 
 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
 {
     struct octeon_hcd *usb = hcd_to_octeon(hcd);
     unsigned long flags;
 
     spin_lock_irqsave(&usb->lock, flags);
     cvmx_usb_poll(usb);
     spin_unlock_irqrestore(&usb->lock, flags);
     return IRQ_HANDLED;
 }
 
 static int octeon_usb_start(struct usb_hcd *hcd)
 {
     hcd->state = HC_STATE_RUNNING;
     return 0;
 }
 
 static void octeon_usb_stop(struct usb_hcd *hcd)
 {
     hcd->state = HC_STATE_HALT;
 }
 
 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
 {
     struct octeon_hcd *usb = hcd_to_octeon(hcd);
 
     return cvmx_usb_get_frame_number(usb);
 }
 
 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
                   struct urb *urb,
                   gfp_t mem_flags)
 {
     struct octeon_hcd *usb = hcd_to_octeon(hcd);
     struct device *dev = hcd->self.controller;
     struct cvmx_usb_transaction *transaction = NULL;
     struct cvmx_usb_pipe *pipe;
     unsigned long flags;
     struct cvmx_usb_iso_packet *iso_packet;
     struct usb_host_endpoint *ep = urb->ep;
     int rc;
 
     urb->status = 0;
     spin_lock_irqsave(&usb->lock, flags);
 
     rc = usb_hcd_link_urb_to_ep(hcd, urb);
     if (rc) {
         spin_unlock_irqrestore(&usb->lock, flags);
         return rc;
     }
 
     if (!ep->hcpriv) {
         enum cvmx_usb_transfer transfer_type;
         enum cvmx_usb_speed speed;
         int split_device = 0;
         int split_port = 0;
 
         switch (usb_pipetype(urb->pipe)) {
         case PIPE_ISOCHRONOUS:
             transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
             break;
         case PIPE_INTERRUPT:
             transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
             break;
         case PIPE_CONTROL:
             transfer_type = CVMX_USB_TRANSFER_CONTROL;
             break;
         default:
             transfer_type = CVMX_USB_TRANSFER_BULK;
             break;
         }
         switch (urb->dev->speed) {
         case USB_SPEED_LOW:
             speed = CVMX_USB_SPEED_LOW;
             break;
         case USB_SPEED_FULL:
             speed = CVMX_USB_SPEED_FULL;
             break;
         default:
             speed = CVMX_USB_SPEED_HIGH;
             break;
         }
         /*
          * For slow devices on high speed ports we need to find the hub
          * that does the speed translation so we know where to send the
          * split transactions.
          */
         if (speed != CVMX_USB_SPEED_HIGH) {
             /*
              * Start at this device and work our way up the usb
              * tree.
              */
             struct usb_device *dev = urb->dev;
 
             while (dev->parent) {
                 /*
                  * If our parent is high speed then he'll
                  * receive the splits.
                  */
                 if (dev->parent->speed == USB_SPEED_HIGH) {
                     split_device = dev->parent->devnum;
                     split_port = dev->portnum;
                     break;
                 }
                 /*
                  * Move up the tree one level. If we make it all
                  * the way up the tree, then the port must not
                  * be in high speed mode and we don't need a
                  * split.
                  */
                 dev = dev->parent;
             }
         }
         pipe = cvmx_usb_open_pipe(usb, usb_pipedevice(urb->pipe),
                       usb_pipeendpoint(urb->pipe), speed,
                       le16_to_cpu(ep->desc.wMaxPacketSize)
                       & 0x7ff,
                       transfer_type,
                       usb_pipein(urb->pipe) ?
                         CVMX_USB_DIRECTION_IN :
                         CVMX_USB_DIRECTION_OUT,
                       urb->interval,
                       (le16_to_cpu(ep->desc.wMaxPacketSize)
                        >> 11) & 0x3,
                       split_device, split_port);
         if (!pipe) {
             usb_hcd_unlink_urb_from_ep(hcd, urb);
             spin_unlock_irqrestore(&usb->lock, flags);
             dev_dbg(dev, "Failed to create pipe\n");
             return -ENOMEM;
         }
         ep->hcpriv = pipe;
     } else {
         pipe = ep->hcpriv;
     }
 
     switch (usb_pipetype(urb->pipe)) {
     case PIPE_ISOCHRONOUS:
         dev_dbg(dev, "Submit isochronous to %d.%d\n",
             usb_pipedevice(urb->pipe),
             usb_pipeendpoint(urb->pipe));
         /*
          * Allocate a structure to use for our private list of
          * isochronous packets.
          */
         iso_packet = kmalloc_array(urb->number_of_packets,
                        sizeof(struct cvmx_usb_iso_packet),
                        GFP_ATOMIC);
         if (iso_packet) {
             int i;
             /* Fill the list with the data from the URB */
             for (i = 0; i < urb->number_of_packets; i++) {
                 iso_packet[i].offset =
                     urb->iso_frame_desc[i].offset;
                 iso_packet[i].length =
                     urb->iso_frame_desc[i].length;
                 iso_packet[i].status = CVMX_USB_STATUS_ERROR;
             }
             /*
              * Store a pointer to the list in the URB setup_packet
              * field. We know this currently isn't being used and
              * this saves us a bunch of logic.
              */
             urb->setup_packet = (char *)iso_packet;
             transaction = cvmx_usb_submit_isochronous(usb,
                                   pipe, urb);
             /*
              * If submit failed we need to free our private packet
              * list.
              */
             if (!transaction) {
                 urb->setup_packet = NULL;
                 kfree(iso_packet);
             }
         }
         break;
     case PIPE_INTERRUPT:
         dev_dbg(dev, "Submit interrupt to %d.%d\n",
             usb_pipedevice(urb->pipe),
             usb_pipeendpoint(urb->pipe));
         transaction = cvmx_usb_submit_interrupt(usb, pipe, urb);
         break;
     case PIPE_CONTROL:
         dev_dbg(dev, "Submit control to %d.%d\n",
             usb_pipedevice(urb->pipe),
             usb_pipeendpoint(urb->pipe));
         transaction = cvmx_usb_submit_control(usb, pipe, urb);
         break;
     case PIPE_BULK:
         dev_dbg(dev, "Submit bulk to %d.%d\n",
             usb_pipedevice(urb->pipe),
             usb_pipeendpoint(urb->pipe));
         transaction = cvmx_usb_submit_bulk(usb, pipe, urb);
         break;
     }
     if (!transaction) {
         usb_hcd_unlink_urb_from_ep(hcd, urb);
         spin_unlock_irqrestore(&usb->lock, flags);
         dev_dbg(dev, "Failed to submit\n");
         return -ENOMEM;
     }
     urb->hcpriv = transaction;
     spin_unlock_irqrestore(&usb->lock, flags);
     return 0;
 }
 
 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
                   struct urb *urb,
                   int status)
 {
     struct octeon_hcd *usb = hcd_to_octeon(hcd);
     unsigned long flags;
     int rc;
 
     if (!urb->dev)
         return -EINVAL;
 
     spin_lock_irqsave(&usb->lock, flags);
 
     rc = usb_hcd_check_unlink_urb(hcd, urb, status);
     if (rc)
         goto out;
 
     urb->status = status;
     cvmx_usb_cancel(usb, urb->ep->hcpriv, urb->hcpriv);
 
 out:
     spin_unlock_irqrestore(&usb->lock, flags);
 
     return rc;
 }
 
 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
                     struct usb_host_endpoint *ep)
 {
     struct device *dev = hcd->self.controller;
 
     if (ep->hcpriv) {
         struct octeon_hcd *usb = hcd_to_octeon(hcd);
         struct cvmx_usb_pipe *pipe = ep->hcpriv;
         unsigned long flags;
 
         spin_lock_irqsave(&usb->lock, flags);
         cvmx_usb_cancel_all(usb, pipe);
         if (cvmx_usb_close_pipe(usb, pipe))
             dev_dbg(dev, "Closing pipe %p failed\n", pipe);
         spin_unlock_irqrestore(&usb->lock, flags);
         ep->hcpriv = NULL;
     }
 }
 
 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
 {
     struct octeon_hcd *usb = hcd_to_octeon(hcd);
     struct cvmx_usb_port_status port_status;
     unsigned long flags;
 
     spin_lock_irqsave(&usb->lock, flags);
     port_status = cvmx_usb_get_status(usb);
     spin_unlock_irqrestore(&usb->lock, flags);
     buf[0] = port_status.connect_change << 1;
 
     return buf[0] != 0;
 }
 
 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
                   u16 wIndex, char *buf, u16 wLength)
 {
     struct octeon_hcd *usb = hcd_to_octeon(hcd);
     struct device *dev = hcd->self.controller;
     struct cvmx_usb_port_status usb_port_status;
     int port_status;
     struct usb_hub_descriptor *desc;
     unsigned long flags;
 
     switch (typeReq) {
     case ClearHubFeature:
         dev_dbg(dev, "ClearHubFeature\n");
         switch (wValue) {
         case C_HUB_LOCAL_POWER:
         case C_HUB_OVER_CURRENT:
             /* Nothing required here */
             break;
         default:
             return -EINVAL;
         }
         break;
     case ClearPortFeature:
         dev_dbg(dev, "ClearPortFeature\n");
         if (wIndex != 1) {
             dev_dbg(dev, " INVALID\n");
             return -EINVAL;
         }
 
         switch (wValue) {
         case USB_PORT_FEAT_ENABLE:
             dev_dbg(dev, " ENABLE\n");
             spin_lock_irqsave(&usb->lock, flags);
             cvmx_usb_disable(usb);
             spin_unlock_irqrestore(&usb->lock, flags);
             break;
         case USB_PORT_FEAT_SUSPEND:
             dev_dbg(dev, " SUSPEND\n");
             /* Not supported on Octeon */
             break;
         case USB_PORT_FEAT_POWER:
             dev_dbg(dev, " POWER\n");
             /* Not supported on Octeon */
             break;
         case USB_PORT_FEAT_INDICATOR:
             dev_dbg(dev, " INDICATOR\n");
             /* Port inidicator not supported */
             break;
         case USB_PORT_FEAT_C_CONNECTION:
             dev_dbg(dev, " C_CONNECTION\n");
             /* Clears drivers internal connect status change flag */
             spin_lock_irqsave(&usb->lock, flags);
             usb->port_status = cvmx_usb_get_status(usb);
             spin_unlock_irqrestore(&usb->lock, flags);
             break;
         case USB_PORT_FEAT_C_RESET:
             dev_dbg(dev, " C_RESET\n");
             /*
              * Clears the driver's internal Port Reset Change flag.
              */
             spin_lock_irqsave(&usb->lock, flags);
             usb->port_status = cvmx_usb_get_status(usb);
             spin_unlock_irqrestore(&usb->lock, flags);
             break;
         case USB_PORT_FEAT_C_ENABLE:
             dev_dbg(dev, " C_ENABLE\n");
             /*
              * Clears the driver's internal Port Enable/Disable
              * Change flag.
              */
             spin_lock_irqsave(&usb->lock, flags);
             usb->port_status = cvmx_usb_get_status(usb);
             spin_unlock_irqrestore(&usb->lock, flags);
             break;
         case USB_PORT_FEAT_C_SUSPEND:
             dev_dbg(dev, " C_SUSPEND\n");
             /*
              * Clears the driver's internal Port Suspend Change
              * flag, which is set when resume signaling on the host
              * port is complete.
              */
             break;
         case USB_PORT_FEAT_C_OVER_CURRENT:
             dev_dbg(dev, " C_OVER_CURRENT\n");
             /* Clears the driver's overcurrent Change flag */
             spin_lock_irqsave(&usb->lock, flags);
             usb->port_status = cvmx_usb_get_status(usb);
             spin_unlock_irqrestore(&usb->lock, flags);
             break;
         default:
             dev_dbg(dev, " UNKNOWN\n");
             return -EINVAL;
         }
         break;
     case GetHubDescriptor:
         dev_dbg(dev, "GetHubDescriptor\n");
         desc = (struct usb_hub_descriptor *)buf;
         desc->bDescLength = 9;
         desc->bDescriptorType = 0x29;
         desc->bNbrPorts = 1;
         desc->wHubCharacteristics = cpu_to_le16(0x08);
         desc->bPwrOn2PwrGood = 1;
         desc->bHubContrCurrent = 0;
         desc->u.hs.DeviceRemovable[0] = 0;
         desc->u.hs.DeviceRemovable[1] = 0xff;
         break;
     case GetHubStatus:
         dev_dbg(dev, "GetHubStatus\n");
         *(__le32 *)buf = 0;
         break;
     case GetPortStatus:
         dev_dbg(dev, "GetPortStatus\n");
         if (wIndex != 1) {
             dev_dbg(dev, " INVALID\n");
             return -EINVAL;
         }
 
         spin_lock_irqsave(&usb->lock, flags);
         usb_port_status = cvmx_usb_get_status(usb);
         spin_unlock_irqrestore(&usb->lock, flags);
         port_status = 0;
 
         if (usb_port_status.connect_change) {
             port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
             dev_dbg(dev, " C_CONNECTION\n");
         }
 
         if (usb_port_status.port_enabled) {
             port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
             dev_dbg(dev, " C_ENABLE\n");
         }
 
         if (usb_port_status.connected) {
             port_status |= (1 << USB_PORT_FEAT_CONNECTION);
             dev_dbg(dev, " CONNECTION\n");
         }
 
         if (usb_port_status.port_enabled) {
             port_status |= (1 << USB_PORT_FEAT_ENABLE);
             dev_dbg(dev, " ENABLE\n");
         }
 
         if (usb_port_status.port_over_current) {
             port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
             dev_dbg(dev, " OVER_CURRENT\n");
         }
 
         if (usb_port_status.port_powered) {
             port_status |= (1 << USB_PORT_FEAT_POWER);
             dev_dbg(dev, " POWER\n");
         }
 
         if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
             port_status |= USB_PORT_STAT_HIGH_SPEED;
             dev_dbg(dev, " HIGHSPEED\n");
         } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
             port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
             dev_dbg(dev, " LOWSPEED\n");
         }
 
         *((__le32 *)buf) = cpu_to_le32(port_status);
         break;
     case SetHubFeature:
         dev_dbg(dev, "SetHubFeature\n");
         /* No HUB features supported */
         break;
     case SetPortFeature:
         dev_dbg(dev, "SetPortFeature\n");
         if (wIndex != 1) {
             dev_dbg(dev, " INVALID\n");
             return -EINVAL;
         }
 
         switch (wValue) {
         case USB_PORT_FEAT_SUSPEND:
             dev_dbg(dev, " SUSPEND\n");
             return -EINVAL;
         case USB_PORT_FEAT_POWER:
             dev_dbg(dev, " POWER\n");
             /*
              * Program the port power bit to drive VBUS on the USB.
              */
             spin_lock_irqsave(&usb->lock, flags);
             USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
                     cvmx_usbcx_hprt, prtpwr, 1);
             spin_unlock_irqrestore(&usb->lock, flags);
             return 0;
         case USB_PORT_FEAT_RESET:
             dev_dbg(dev, " RESET\n");
             spin_lock_irqsave(&usb->lock, flags);
             cvmx_usb_reset_port(usb);
             spin_unlock_irqrestore(&usb->lock, flags);
             return 0;
         case USB_PORT_FEAT_INDICATOR:
             dev_dbg(dev, " INDICATOR\n");
             /* Not supported */
             break;
         default:
             dev_dbg(dev, " UNKNOWN\n");
             return -EINVAL;
         }
         break;
     default:
         dev_dbg(dev, "Unknown root hub request\n");
         return -EINVAL;
     }
     return 0;
 }
 
 static const struct hc_driver octeon_hc_driver = {
     .description        = "Octeon USB",
     .product_desc       = "Octeon Host Controller",
     .hcd_priv_size      = sizeof(struct octeon_hcd),
     .irq            = octeon_usb_irq,
     .flags          = HCD_MEMORY | HCD_DMA | HCD_USB2,
     .start          = octeon_usb_start,
     .stop           = octeon_usb_stop,
     .urb_enqueue        = octeon_usb_urb_enqueue,
     .urb_dequeue        = octeon_usb_urb_dequeue,
     .endpoint_disable   = octeon_usb_endpoint_disable,
     .get_frame_number   = octeon_usb_get_frame_number,
     .hub_status_data    = octeon_usb_hub_status_data,
     .hub_control        = octeon_usb_hub_control,
     .map_urb_for_dma    = octeon_map_urb_for_dma,
     .unmap_urb_for_dma  = octeon_unmap_urb_for_dma,
 };
 
 static int octeon_usb_probe(struct platform_device *pdev)
 {
     int status;
     int initialize_flags;
     int usb_num;
     struct resource *res_mem;
     struct device_node *usbn_node;
     int irq = platform_get_irq(pdev, 0);
     struct device *dev = &pdev->dev;
     struct octeon_hcd *usb;
     struct usb_hcd *hcd;
     u32 clock_rate = 48000000;
     bool is_crystal_clock = false;
     const char *clock_type;
     int i;
 
     if (!dev->of_node) {
         dev_err(dev, "Error: empty of_node\n");
         return -ENXIO;
     }
     usbn_node = dev->of_node->parent;
 
     i = of_property_read_u32(usbn_node,
                  "clock-frequency", &clock_rate);
     if (i)
         i = of_property_read_u32(usbn_node,
                      "refclk-frequency", &clock_rate);
     if (i) {
         dev_err(dev, "No USBN \"clock-frequency\"\n");
         return -ENXIO;
     }
     switch (clock_rate) {
     case 12000000:
         initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
         break;
     case 24000000:
         initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
         break;
     case 48000000:
         initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
         break;
     default:
         dev_err(dev, "Illegal USBN \"clock-frequency\" %u\n",
             clock_rate);
         return -ENXIO;
     }
 
     i = of_property_read_string(usbn_node,
                     "cavium,refclk-type", &clock_type);
     if (i)
         i = of_property_read_string(usbn_node,
                         "refclk-type", &clock_type);
 
     if (!i && strcmp("crystal", clock_type) == 0)
         is_crystal_clock = true;
 
     if (is_crystal_clock)
         initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
     else
         initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
 
     res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
     if (!res_mem) {
         dev_err(dev, "found no memory resource\n");
         return -ENXIO;
     }
     usb_num = (res_mem->start >> 44) & 1;
 
     if (irq < 0) {
         /* Defective device tree, but we know how to fix it. */
         irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
 
         irq = irq_create_mapping(NULL, hwirq);
     }
 
     /*
      * Set the DMA mask to 64bits so we get buffers already translated for
      * DMA.
      */
     i = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(64));
     if (i)
         return i;
 
     /*
      * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
      * IOB priority registers.  Under heavy network load USB
      * hardware can be starved by the IOB causing a crash.  Give
      * it a priority boost if it has been waiting more than 400
      * cycles to avoid this situation.
      *
      * Testing indicates that a cnt_val of 8192 is not sufficient,
      * but no failures are seen with 4096.  We choose a value of
      * 400 to give a safety factor of 10.
      */
     if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
         union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
 
         pri_cnt.u64 = 0;
         pri_cnt.s.cnt_enb = 1;
         pri_cnt.s.cnt_val = 400;
         cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
     }
 
     hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
     if (!hcd) {
         dev_dbg(dev, "Failed to allocate memory for HCD\n");
         return -1;
     }
     hcd->uses_new_polling = 1;
     usb = (struct octeon_hcd *)hcd->hcd_priv;
 
     spin_lock_init(&usb->lock);
 
     usb->init_flags = initialize_flags;
 
     /* Initialize the USB state structure */
     usb->index = usb_num;
     INIT_LIST_HEAD(&usb->idle_pipes);
     for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
         INIT_LIST_HEAD(&usb->active_pipes[i]);
 
     /* Due to an errata, CN31XX doesn't support DMA */
     if (OCTEON_IS_MODEL(OCTEON_CN31XX)) {
         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
         /* Only use one channel with non DMA */
         usb->idle_hardware_channels = 0x1;
     } else if (OCTEON_IS_MODEL(OCTEON_CN5XXX)) {
         /* CN5XXX have an errata with channel 3 */
         usb->idle_hardware_channels = 0xf7;
     } else {
         usb->idle_hardware_channels = 0xff;
     }
 
     status = cvmx_usb_initialize(dev, usb);
     if (status) {
         dev_dbg(dev, "USB initialization failed with %d\n", status);
         usb_put_hcd(hcd);
         return -1;
     }
 
     status = usb_add_hcd(hcd, irq, 0);
     if (status) {
         dev_dbg(dev, "USB add HCD failed with %d\n", status);
         usb_put_hcd(hcd);
         return -1;
     }
     device_wakeup_enable(hcd->self.controller);
 
     dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
 
     return 0;
 }
 
 static int octeon_usb_remove(struct platform_device *pdev)
 {
     int status;
     struct device *dev = &pdev->dev;
     struct usb_hcd *hcd = dev_get_drvdata(dev);
     struct octeon_hcd *usb = hcd_to_octeon(hcd);
     unsigned long flags;
 
     usb_remove_hcd(hcd);
     spin_lock_irqsave(&usb->lock, flags);
     status = cvmx_usb_shutdown(usb);
     spin_unlock_irqrestore(&usb->lock, flags);
     if (status)
         dev_dbg(dev, "USB shutdown failed with %d\n", status);
 
     usb_put_hcd(hcd);
 
     return 0;
 }
 
 static const struct of_device_id octeon_usb_match[] = {
     {
         .compatible = "cavium,octeon-5750-usbc",
     },
     {},
 };
 MODULE_DEVICE_TABLE(of, octeon_usb_match);
 
 static struct platform_driver octeon_usb_driver = {
     .driver = {
         .name       = "octeon-hcd",
         .of_match_table = octeon_usb_match,
     },
     .probe      = octeon_usb_probe,
     .remove     = octeon_usb_remove,
 };
 
 static int __init octeon_usb_driver_init(void)
 {
     if (usb_disabled())
         return 0;
 
     return platform_driver_register(&octeon_usb_driver);
 }
 module_init(octeon_usb_driver_init);
 
 static void __exit octeon_usb_driver_exit(void)
 {
     if (usb_disabled())
         return;
 
     platform_driver_unregister(&octeon_usb_driver);
 }
 module_exit(octeon_usb_driver_exit);
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");