OSCL-LXR linux-6.0.1/​drivers/​mailbox/​bcm-flexrm-mailbox.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0-only
 // Copyright (C) 2017 Broadcom
 
 /*
  * Broadcom FlexRM Mailbox Driver
  *
  * Each Broadcom FlexSparx4 offload engine is implemented as an
  * extension to Broadcom FlexRM ring manager. The FlexRM ring
  * manager provides a set of rings which can be used to submit
  * work to a FlexSparx4 offload engine.
  *
  * This driver creates a mailbox controller using a set of FlexRM
  * rings where each mailbox channel represents a separate FlexRM ring.
  */
 
 #include <asm/barrier.h>
 #include <asm/byteorder.h>
 #include <linux/atomic.h>
 #include <linux/bitmap.h>
 #include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmapool.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/mailbox_controller.h>
 #include <linux/mailbox_client.h>
 #include <linux/mailbox/brcm-message.h>
 #include <linux/module.h>
 #include <linux/msi.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
 
 /* ====== FlexRM register defines ===== */
 
 /* FlexRM configuration */
 #define RING_REGS_SIZE                  0x10000
 #define RING_DESC_SIZE                  8
 #define RING_DESC_INDEX(offset)             \
             ((offset) / RING_DESC_SIZE)
 #define RING_DESC_OFFSET(index)             \
             ((index) * RING_DESC_SIZE)
 #define RING_MAX_REQ_COUNT              1024
 #define RING_BD_ALIGN_ORDER             12
 #define RING_BD_ALIGN_CHECK(addr)           \
             (!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1)))
 #define RING_BD_TOGGLE_INVALID(offset)          \
             (((offset) >> RING_BD_ALIGN_ORDER) & 0x1)
 #define RING_BD_TOGGLE_VALID(offset)            \
             (!RING_BD_TOGGLE_INVALID(offset))
 #define RING_BD_DESC_PER_REQ                32
 #define RING_BD_DESC_COUNT              \
             (RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ)
 #define RING_BD_SIZE                    \
             (RING_BD_DESC_COUNT * RING_DESC_SIZE)
 #define RING_CMPL_ALIGN_ORDER               13
 #define RING_CMPL_DESC_COUNT                RING_MAX_REQ_COUNT
 #define RING_CMPL_SIZE                  \
             (RING_CMPL_DESC_COUNT * RING_DESC_SIZE)
 #define RING_VER_MAGIC                  0x76303031
 
 /* Per-Ring register offsets */
 #define RING_VER                    0x000
 #define RING_BD_START_ADDR              0x004
 #define RING_BD_READ_PTR                0x008
 #define RING_BD_WRITE_PTR               0x00c
 #define RING_BD_READ_PTR_DDR_LS             0x010
 #define RING_BD_READ_PTR_DDR_MS             0x014
 #define RING_CMPL_START_ADDR                0x018
 #define RING_CMPL_WRITE_PTR             0x01c
 #define RING_NUM_REQ_RECV_LS                0x020
 #define RING_NUM_REQ_RECV_MS                0x024
 #define RING_NUM_REQ_TRANS_LS               0x028
 #define RING_NUM_REQ_TRANS_MS               0x02c
 #define RING_NUM_REQ_OUTSTAND               0x030
 #define RING_CONTROL                    0x034
 #define RING_FLUSH_DONE                 0x038
 #define RING_MSI_ADDR_LS                0x03c
 #define RING_MSI_ADDR_MS                0x040
 #define RING_MSI_CONTROL                0x048
 #define RING_BD_READ_PTR_DDR_CONTROL            0x04c
 #define RING_MSI_DATA_VALUE             0x064
 
 /* Register RING_BD_START_ADDR fields */
 #define BD_LAST_UPDATE_HW_SHIFT             28
 #define BD_LAST_UPDATE_HW_MASK              0x1
 #define BD_START_ADDR_VALUE(pa)             \
     ((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
 #define BD_START_ADDR_DECODE(val)           \
     ((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
 
 /* Register RING_CMPL_START_ADDR fields */
 #define CMPL_START_ADDR_VALUE(pa)           \
     ((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
 
 /* Register RING_CONTROL fields */
 #define CONTROL_MASK_DISABLE_CONTROL            12
 #define CONTROL_FLUSH_SHIFT             5
 #define CONTROL_ACTIVE_SHIFT                4
 #define CONTROL_RATE_ADAPT_MASK             0xf
 #define CONTROL_RATE_DYNAMIC                0x0
 #define CONTROL_RATE_FAST               0x8
 #define CONTROL_RATE_MEDIUM             0x9
 #define CONTROL_RATE_SLOW               0xa
 #define CONTROL_RATE_IDLE               0xb
 
 /* Register RING_FLUSH_DONE fields */
 #define FLUSH_DONE_MASK                 0x1
 
 /* Register RING_MSI_CONTROL fields */
 #define MSI_TIMER_VAL_SHIFT             16
 #define MSI_TIMER_VAL_MASK              0xffff
 #define MSI_ENABLE_SHIFT                15
 #define MSI_ENABLE_MASK                 0x1
 #define MSI_COUNT_SHIFT                 0
 #define MSI_COUNT_MASK                  0x3ff
 
 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT         16
 #define BD_READ_PTR_DDR_TIMER_VAL_MASK          0xffff
 #define BD_READ_PTR_DDR_ENABLE_SHIFT            15
 #define BD_READ_PTR_DDR_ENABLE_MASK         0x1
 
 /* ====== FlexRM ring descriptor defines ===== */
 
 /* Completion descriptor format */
 #define CMPL_OPAQUE_SHIFT           0
 #define CMPL_OPAQUE_MASK            0xffff
 #define CMPL_ENGINE_STATUS_SHIFT        16
 #define CMPL_ENGINE_STATUS_MASK         0xffff
 #define CMPL_DME_STATUS_SHIFT           32
 #define CMPL_DME_STATUS_MASK            0xffff
 #define CMPL_RM_STATUS_SHIFT            48
 #define CMPL_RM_STATUS_MASK         0xffff
 
 /* Completion DME status code */
 #define DME_STATUS_MEM_COR_ERR          BIT(0)
 #define DME_STATUS_MEM_UCOR_ERR         BIT(1)
 #define DME_STATUS_FIFO_UNDERFLOW       BIT(2)
 #define DME_STATUS_FIFO_OVERFLOW        BIT(3)
 #define DME_STATUS_RRESP_ERR            BIT(4)
 #define DME_STATUS_BRESP_ERR            BIT(5)
 #define DME_STATUS_ERROR_MASK           (DME_STATUS_MEM_COR_ERR | \
                          DME_STATUS_MEM_UCOR_ERR | \
                          DME_STATUS_FIFO_UNDERFLOW | \
                          DME_STATUS_FIFO_OVERFLOW | \
                          DME_STATUS_RRESP_ERR | \
                          DME_STATUS_BRESP_ERR)
 
 /* Completion RM status code */
 #define RM_STATUS_CODE_SHIFT            0
 #define RM_STATUS_CODE_MASK         0x3ff
 #define RM_STATUS_CODE_GOOD         0x0
 #define RM_STATUS_CODE_AE_TIMEOUT       0x3ff
 
 /* General descriptor format */
 #define DESC_TYPE_SHIFT             60
 #define DESC_TYPE_MASK              0xf
 #define DESC_PAYLOAD_SHIFT          0
 #define DESC_PAYLOAD_MASK           0x0fffffffffffffff
 
 /* Null descriptor format  */
 #define NULL_TYPE               0
 #define NULL_TOGGLE_SHIFT           58
 #define NULL_TOGGLE_MASK            0x1
 
 /* Header descriptor format */
 #define HEADER_TYPE             1
 #define HEADER_TOGGLE_SHIFT         58
 #define HEADER_TOGGLE_MASK          0x1
 #define HEADER_ENDPKT_SHIFT         57
 #define HEADER_ENDPKT_MASK          0x1
 #define HEADER_STARTPKT_SHIFT           56
 #define HEADER_STARTPKT_MASK            0x1
 #define HEADER_BDCOUNT_SHIFT            36
 #define HEADER_BDCOUNT_MASK         0x1f
 #define HEADER_BDCOUNT_MAX          HEADER_BDCOUNT_MASK
 #define HEADER_FLAGS_SHIFT          16
 #define HEADER_FLAGS_MASK           0xffff
 #define HEADER_OPAQUE_SHIFT         0
 #define HEADER_OPAQUE_MASK          0xffff
 
 /* Source (SRC) descriptor format */
 #define SRC_TYPE                2
 #define SRC_LENGTH_SHIFT            44
 #define SRC_LENGTH_MASK             0xffff
 #define SRC_ADDR_SHIFT              0
 #define SRC_ADDR_MASK               0x00000fffffffffff
 
 /* Destination (DST) descriptor format */
 #define DST_TYPE                3
 #define DST_LENGTH_SHIFT            44
 #define DST_LENGTH_MASK             0xffff
 #define DST_ADDR_SHIFT              0
 #define DST_ADDR_MASK               0x00000fffffffffff
 
 /* Immediate (IMM) descriptor format */
 #define IMM_TYPE                4
 #define IMM_DATA_SHIFT              0
 #define IMM_DATA_MASK               0x0fffffffffffffff
 
 /* Next pointer (NPTR) descriptor format */
 #define NPTR_TYPE               5
 #define NPTR_TOGGLE_SHIFT           58
 #define NPTR_TOGGLE_MASK            0x1
 #define NPTR_ADDR_SHIFT             0
 #define NPTR_ADDR_MASK              0x00000fffffffffff
 
 /* Mega source (MSRC) descriptor format */
 #define MSRC_TYPE               6
 #define MSRC_LENGTH_SHIFT           44
 #define MSRC_LENGTH_MASK            0xffff
 #define MSRC_ADDR_SHIFT             0
 #define MSRC_ADDR_MASK              0x00000fffffffffff
 
 /* Mega destination (MDST) descriptor format */
 #define MDST_TYPE               7
 #define MDST_LENGTH_SHIFT           44
 #define MDST_LENGTH_MASK            0xffff
 #define MDST_ADDR_SHIFT             0
 #define MDST_ADDR_MASK              0x00000fffffffffff
 
 /* Source with tlast (SRCT) descriptor format */
 #define SRCT_TYPE               8
 #define SRCT_LENGTH_SHIFT           44
 #define SRCT_LENGTH_MASK            0xffff
 #define SRCT_ADDR_SHIFT             0
 #define SRCT_ADDR_MASK              0x00000fffffffffff
 
 /* Destination with tlast (DSTT) descriptor format */
 #define DSTT_TYPE               9
 #define DSTT_LENGTH_SHIFT           44
 #define DSTT_LENGTH_MASK            0xffff
 #define DSTT_ADDR_SHIFT             0
 #define DSTT_ADDR_MASK              0x00000fffffffffff
 
 /* Immediate with tlast (IMMT) descriptor format */
 #define IMMT_TYPE               10
 #define IMMT_DATA_SHIFT             0
 #define IMMT_DATA_MASK              0x0fffffffffffffff
 
 /* Descriptor helper macros */
 #define DESC_DEC(_d, _s, _m)            (((_d) >> (_s)) & (_m))
 #define DESC_ENC(_d, _v, _s, _m)        \
             do { \
                 (_d) &= ~((u64)(_m) << (_s)); \
                 (_d) |= (((u64)(_v) & (_m)) << (_s)); \
             } while (0)
 
 /* ====== FlexRM data structures ===== */
 
 struct flexrm_ring {
     /* Unprotected members */
     int num;
     struct flexrm_mbox *mbox;
     void __iomem *regs;
     bool irq_requested;
     unsigned int irq;
     cpumask_t irq_aff_hint;
     unsigned int msi_timer_val;
     unsigned int msi_count_threshold;
     struct brcm_message *requests[RING_MAX_REQ_COUNT];
     void *bd_base;
     dma_addr_t bd_dma_base;
     u32 bd_write_offset;
     void *cmpl_base;
     dma_addr_t cmpl_dma_base;
     /* Atomic stats */
     atomic_t msg_send_count;
     atomic_t msg_cmpl_count;
     /* Protected members */
     spinlock_t lock;
     DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);
     u32 cmpl_read_offset;
 };
 
 struct flexrm_mbox {
     struct device *dev;
     void __iomem *regs;
     u32 num_rings;
     struct flexrm_ring *rings;
     struct dma_pool *bd_pool;
     struct dma_pool *cmpl_pool;
     struct dentry *root;
     struct mbox_controller controller;
 };
 
 /* ====== FlexRM ring descriptor helper routines ===== */
 
 static u64 flexrm_read_desc(void *desc_ptr)
 {
     return le64_to_cpu(*((u64 *)desc_ptr));
 }
 
 static void flexrm_write_desc(void *desc_ptr, u64 desc)
 {
     *((u64 *)desc_ptr) = cpu_to_le64(desc);
 }
 
 static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc)
 {
     return (u32)(cmpl_desc & CMPL_OPAQUE_MASK);
 }
 
 static int flexrm_cmpl_desc_to_error(u64 cmpl_desc)
 {
     u32 status;
 
     status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT,
               CMPL_DME_STATUS_MASK);
     if (status & DME_STATUS_ERROR_MASK)
         return -EIO;
 
     status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT,
               CMPL_RM_STATUS_MASK);
     status &= RM_STATUS_CODE_MASK;
     if (status == RM_STATUS_CODE_AE_TIMEOUT)
         return -ETIMEDOUT;
 
     return 0;
 }
 
 static bool flexrm_is_next_table_desc(void *desc_ptr)
 {
     u64 desc = flexrm_read_desc(desc_ptr);
     u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 
     return (type == NPTR_TYPE) ? true : false;
 }
 
 static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK);
     DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK);
 
     return desc;
 }
 
 static u64 flexrm_null_desc(u32 toggle)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK);
 
     return desc;
 }
 
 static u32 flexrm_estimate_header_desc_count(u32 nhcnt)
 {
     u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX;
 
     if (!(nhcnt % HEADER_BDCOUNT_MAX))
         hcnt += 1;
 
     return hcnt;
 }
 
 static void flexrm_flip_header_toggle(void *desc_ptr)
 {
     u64 desc = flexrm_read_desc(desc_ptr);
 
     if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT))
         desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT);
     else
         desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT);
 
     flexrm_write_desc(desc_ptr, desc);
 }
 
 static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt,
                    u32 bdcount, u32 flags, u32 opaque)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK);
     DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK);
     DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK);
     DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK);
     DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK);
     DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK);
 
     return desc;
 }
 
 static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid,
                  u64 desc, void **desc_ptr, u32 *toggle,
                  void *start_desc, void *end_desc)
 {
     u64 d;
     u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount;
 
     /* Sanity check */
     if (nhcnt <= nhpos)
         return;
 
     /*
      * Each request or packet start with a HEADER descriptor followed
      * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
      * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
      * following a HEADER descriptor is represented by BDCOUNT field
      * of HEADER descriptor. The max value of BDCOUNT field is 31 which
      * means we can only have 31 non-HEADER descriptors following one
      * HEADER descriptor.
      *
      * In general use, number of non-HEADER descriptors can easily go
      * beyond 31. To tackle this situation, we have packet (or request)
      * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor.
      *
      * To use packet extension, the first HEADER descriptor of request
      * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
      * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
      * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
      * TOGGLE bit of the first HEADER will be set to invalid state to
      * ensure that FlexRM does not start fetching descriptors till all
      * descriptors are enqueued. The user of this function will flip
      * the TOGGLE bit of first HEADER after all descriptors are
      * enqueued.
      */
 
     if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
         /* Prepare the header descriptor */
         nhavail = (nhcnt - nhpos);
         _toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
         _startpkt = (nhpos == 0) ? 0x1 : 0x0;
         _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
         _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
                 nhavail : HEADER_BDCOUNT_MAX;
         if (nhavail <= HEADER_BDCOUNT_MAX)
             _bdcount = nhavail;
         else
             _bdcount = HEADER_BDCOUNT_MAX;
         d = flexrm_header_desc(_toggle, _startpkt, _endpkt,
                     _bdcount, 0x0, reqid);
 
         /* Write header descriptor */
         flexrm_write_desc(*desc_ptr, d);
 
         /* Point to next descriptor */
         *desc_ptr += sizeof(desc);
         if (*desc_ptr == end_desc)
             *desc_ptr = start_desc;
 
         /* Skip next pointer descriptors */
         while (flexrm_is_next_table_desc(*desc_ptr)) {
             *toggle = (*toggle) ? 0 : 1;
             *desc_ptr += sizeof(desc);
             if (*desc_ptr == end_desc)
                 *desc_ptr = start_desc;
         }
     }
 
     /* Write desired descriptor */
     flexrm_write_desc(*desc_ptr, desc);
 
     /* Point to next descriptor */
     *desc_ptr += sizeof(desc);
     if (*desc_ptr == end_desc)
         *desc_ptr = start_desc;
 
     /* Skip next pointer descriptors */
     while (flexrm_is_next_table_desc(*desc_ptr)) {
         *toggle = (*toggle) ? 0 : 1;
         *desc_ptr += sizeof(desc);
         if (*desc_ptr == end_desc)
             *desc_ptr = start_desc;
     }
 }
 
 static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK);
     DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK);
 
     return desc;
 }
 
 static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK);
     DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK);
 
     return desc;
 }
 
 static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK);
     DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK);
 
     return desc;
 }
 
 static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK);
     DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK);
 
     return desc;
 }
 
 static u64 flexrm_imm_desc(u64 data)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK);
 
     return desc;
 }
 
 static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK);
     DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK);
 
     return desc;
 }
 
 static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK);
     DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK);
 
     return desc;
 }
 
 static u64 flexrm_immt_desc(u64 data)
 {
     u64 desc = 0;
 
     DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
     DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK);
 
     return desc;
 }
 
 static bool flexrm_spu_sanity_check(struct brcm_message *msg)
 {
     struct scatterlist *sg;
 
     if (!msg->spu.src || !msg->spu.dst)
         return false;
     for (sg = msg->spu.src; sg; sg = sg_next(sg)) {
         if (sg->length & 0xf) {
             if (sg->length > SRC_LENGTH_MASK)
                 return false;
         } else {
             if (sg->length > (MSRC_LENGTH_MASK * 16))
                 return false;
         }
     }
     for (sg = msg->spu.dst; sg; sg = sg_next(sg)) {
         if (sg->length & 0xf) {
             if (sg->length > DST_LENGTH_MASK)
                 return false;
         } else {
             if (sg->length > (MDST_LENGTH_MASK * 16))
                 return false;
         }
     }
 
     return true;
 }
 
 static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg)
 {
     u32 cnt = 0;
     unsigned int dst_target = 0;
     struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
 
     while (src_sg || dst_sg) {
         if (src_sg) {
             cnt++;
             dst_target = src_sg->length;
             src_sg = sg_next(src_sg);
         } else
             dst_target = UINT_MAX;
 
         while (dst_target && dst_sg) {
             cnt++;
             if (dst_sg->length < dst_target)
                 dst_target -= dst_sg->length;
             else
                 dst_target = 0;
             dst_sg = sg_next(dst_sg);
         }
     }
 
     return cnt;
 }
 
 static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg)
 {
     int rc;
 
     rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
             DMA_TO_DEVICE);
     if (rc < 0)
         return rc;
 
     rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
             DMA_FROM_DEVICE);
     if (rc < 0) {
         dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
                  DMA_TO_DEVICE);
         return rc;
     }
 
     return 0;
 }
 
 static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg)
 {
     dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
              DMA_FROM_DEVICE);
     dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
              DMA_TO_DEVICE);
 }
 
 static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt,
                      u32 reqid, void *desc_ptr, u32 toggle,
                      void *start_desc, void *end_desc)
 {
     u64 d;
     u32 nhpos = 0;
     void *orig_desc_ptr = desc_ptr;
     unsigned int dst_target = 0;
     struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
 
     while (src_sg || dst_sg) {
         if (src_sg) {
             if (sg_dma_len(src_sg) & 0xf)
                 d = flexrm_src_desc(sg_dma_address(src_sg),
                              sg_dma_len(src_sg));
             else
                 d = flexrm_msrc_desc(sg_dma_address(src_sg),
                               sg_dma_len(src_sg)/16);
             flexrm_enqueue_desc(nhpos, nhcnt, reqid,
                          d, &desc_ptr, &toggle,
                          start_desc, end_desc);
             nhpos++;
             dst_target = sg_dma_len(src_sg);
             src_sg = sg_next(src_sg);
         } else
             dst_target = UINT_MAX;
 
         while (dst_target && dst_sg) {
             if (sg_dma_len(dst_sg) & 0xf)
                 d = flexrm_dst_desc(sg_dma_address(dst_sg),
                              sg_dma_len(dst_sg));
             else
                 d = flexrm_mdst_desc(sg_dma_address(dst_sg),
                               sg_dma_len(dst_sg)/16);
             flexrm_enqueue_desc(nhpos, nhcnt, reqid,
                          d, &desc_ptr, &toggle,
                          start_desc, end_desc);
             nhpos++;
             if (sg_dma_len(dst_sg) < dst_target)
                 dst_target -= sg_dma_len(dst_sg);
             else
                 dst_target = 0;
             dst_sg = sg_next(dst_sg);
         }
     }
 
     /* Null descriptor with invalid toggle bit */
     flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
 
     /* Ensure that descriptors have been written to memory */
     wmb();
 
     /* Flip toggle bit in header */
     flexrm_flip_header_toggle(orig_desc_ptr);
 
     return desc_ptr;
 }
 
 static bool flexrm_sba_sanity_check(struct brcm_message *msg)
 {
     u32 i;
 
     if (!msg->sba.cmds || !msg->sba.cmds_count)
         return false;
 
     for (i = 0; i < msg->sba.cmds_count; i++) {
         if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
              (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) &&
             (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT))
             return false;
         if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) &&
             (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
             return false;
         if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) &&
             (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
             return false;
         if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) &&
             (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK))
             return false;
         if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) &&
             (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK))
             return false;
     }
 
     return true;
 }
 
 static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg)
 {
     u32 i, cnt;
 
     cnt = 0;
     for (i = 0; i < msg->sba.cmds_count; i++) {
         cnt++;
 
         if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
             (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C))
             cnt++;
 
         if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP)
             cnt++;
 
         if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)
             cnt++;
     }
 
     return cnt;
 }
 
 static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt,
                      u32 reqid, void *desc_ptr, u32 toggle,
                      void *start_desc, void *end_desc)
 {
     u64 d;
     u32 i, nhpos = 0;
     struct brcm_sba_command *c;
     void *orig_desc_ptr = desc_ptr;
 
     /* Convert SBA commands into descriptors */
     for (i = 0; i < msg->sba.cmds_count; i++) {
         c = &msg->sba.cmds[i];
 
         if ((c->flags & BRCM_SBA_CMD_HAS_RESP) &&
             (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) {
             /* Destination response descriptor */
             d = flexrm_dst_desc(c->resp, c->resp_len);
             flexrm_enqueue_desc(nhpos, nhcnt, reqid,
                          d, &desc_ptr, &toggle,
                          start_desc, end_desc);
             nhpos++;
         } else if (c->flags & BRCM_SBA_CMD_HAS_RESP) {
             /* Destination response with tlast descriptor */
             d = flexrm_dstt_desc(c->resp, c->resp_len);
             flexrm_enqueue_desc(nhpos, nhcnt, reqid,
                          d, &desc_ptr, &toggle,
                          start_desc, end_desc);
             nhpos++;
         }
 
         if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) {
             /* Destination with tlast descriptor */
             d = flexrm_dstt_desc(c->data, c->data_len);
             flexrm_enqueue_desc(nhpos, nhcnt, reqid,
                          d, &desc_ptr, &toggle,
                          start_desc, end_desc);
             nhpos++;
         }
 
         if (c->flags & BRCM_SBA_CMD_TYPE_B) {
             /* Command as immediate descriptor */
             d = flexrm_imm_desc(c->cmd);
             flexrm_enqueue_desc(nhpos, nhcnt, reqid,
                          d, &desc_ptr, &toggle,
                          start_desc, end_desc);
             nhpos++;
         } else {
             /* Command as immediate descriptor with tlast */
             d = flexrm_immt_desc(c->cmd);
             flexrm_enqueue_desc(nhpos, nhcnt, reqid,
                          d, &desc_ptr, &toggle,
                          start_desc, end_desc);
             nhpos++;
         }
 
         if ((c->flags & BRCM_SBA_CMD_TYPE_B) ||
             (c->flags & BRCM_SBA_CMD_TYPE_C)) {
             /* Source with tlast descriptor */
             d = flexrm_srct_desc(c->data, c->data_len);
             flexrm_enqueue_desc(nhpos, nhcnt, reqid,
                          d, &desc_ptr, &toggle,
                          start_desc, end_desc);
             nhpos++;
         }
     }
 
     /* Null descriptor with invalid toggle bit */
     flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
 
     /* Ensure that descriptors have been written to memory */
     wmb();
 
     /* Flip toggle bit in header */
     flexrm_flip_header_toggle(orig_desc_ptr);
 
     return desc_ptr;
 }
 
 static bool flexrm_sanity_check(struct brcm_message *msg)
 {
     if (!msg)
         return false;
 
     switch (msg->type) {
     case BRCM_MESSAGE_SPU:
         return flexrm_spu_sanity_check(msg);
     case BRCM_MESSAGE_SBA:
         return flexrm_sba_sanity_check(msg);
     default:
         return false;
     };
 }
 
 static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg)
 {
     if (!msg)
         return 0;
 
     switch (msg->type) {
     case BRCM_MESSAGE_SPU:
         return flexrm_spu_estimate_nonheader_desc_count(msg);
     case BRCM_MESSAGE_SBA:
         return flexrm_sba_estimate_nonheader_desc_count(msg);
     default:
         return 0;
     };
 }
 
 static int flexrm_dma_map(struct device *dev, struct brcm_message *msg)
 {
     if (!dev || !msg)
         return -EINVAL;
 
     switch (msg->type) {
     case BRCM_MESSAGE_SPU:
         return flexrm_spu_dma_map(dev, msg);
     default:
         break;
     }
 
     return 0;
 }
 
 static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg)
 {
     if (!dev || !msg)
         return;
 
     switch (msg->type) {
     case BRCM_MESSAGE_SPU:
         flexrm_spu_dma_unmap(dev, msg);
         break;
     default:
         break;
     }
 }
 
 static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,
                 u32 reqid, void *desc_ptr, u32 toggle,
                 void *start_desc, void *end_desc)
 {
     if (!msg || !desc_ptr || !start_desc || !end_desc)
         return ERR_PTR(-ENOTSUPP);
 
     if ((desc_ptr < start_desc) || (end_desc <= desc_ptr))
         return ERR_PTR(-ERANGE);
 
     switch (msg->type) {
     case BRCM_MESSAGE_SPU:
         return flexrm_spu_write_descs(msg, nhcnt, reqid,
                            desc_ptr, toggle,
                            start_desc, end_desc);
     case BRCM_MESSAGE_SBA:
         return flexrm_sba_write_descs(msg, nhcnt, reqid,
                            desc_ptr, toggle,
                            start_desc, end_desc);
     default:
         return ERR_PTR(-ENOTSUPP);
     };
 }
 
 /* ====== FlexRM driver helper routines ===== */
 
 static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,
                        struct seq_file *file)
 {
     int i;
     const char *state;
     struct flexrm_ring *ring;
 
     seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",
            "Ring#", "State", "BD_Addr", "BD_Size",
            "Cmpl_Addr", "Cmpl_Size");
 
     for (i = 0; i < mbox->num_rings; i++) {
         ring = &mbox->rings[i];
         if (readl(ring->regs + RING_CONTROL) &
             BIT(CONTROL_ACTIVE_SHIFT))
             state = "active";
         else
             state = "inactive";
         seq_printf(file,
                "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",
                ring->num, state,
                (unsigned long long)ring->bd_dma_base,
                (u32)RING_BD_SIZE,
                (unsigned long long)ring->cmpl_dma_base,
                (u32)RING_CMPL_SIZE);
     }
 }
 
 static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,
                       struct seq_file *file)
 {
     int i;
     u32 val, bd_read_offset;
     struct flexrm_ring *ring;
 
     seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",
            "Ring#", "BD_Read", "BD_Write",
            "Cmpl_Read", "Submitted", "Completed");
 
     for (i = 0; i < mbox->num_rings; i++) {
         ring = &mbox->rings[i];
         bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
         val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
         bd_read_offset *= RING_DESC_SIZE;
         bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -
                     ring->bd_dma_base);
         seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",
                ring->num,
                (u32)bd_read_offset,
                (u32)ring->bd_write_offset,
                (u32)ring->cmpl_read_offset,
                (u32)atomic_read(&ring->msg_send_count),
                (u32)atomic_read(&ring->msg_cmpl_count));
     }
 }
 
 static int flexrm_new_request(struct flexrm_ring *ring,
                 struct brcm_message *batch_msg,
                 struct brcm_message *msg)
 {
     void *next;
     unsigned long flags;
     u32 val, count, nhcnt;
     u32 read_offset, write_offset;
     bool exit_cleanup = false;
     int ret = 0, reqid;
 
     /* Do sanity check on message */
     if (!flexrm_sanity_check(msg))
         return -EIO;
     msg->error = 0;
 
     /* If no requests possible then save data pointer and goto done. */
     spin_lock_irqsave(&ring->lock, flags);
     reqid = bitmap_find_free_region(ring->requests_bmap,
                     RING_MAX_REQ_COUNT, 0);
     spin_unlock_irqrestore(&ring->lock, flags);
     if (reqid < 0)
         return -ENOSPC;
     ring->requests[reqid] = msg;
 
     /* Do DMA mappings for the message */
     ret = flexrm_dma_map(ring->mbox->dev, msg);
     if (ret < 0) {
         ring->requests[reqid] = NULL;
         spin_lock_irqsave(&ring->lock, flags);
         bitmap_release_region(ring->requests_bmap, reqid, 0);
         spin_unlock_irqrestore(&ring->lock, flags);
         return ret;
     }
 
     /* Determine current HW BD read offset */
     read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
     val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
     read_offset *= RING_DESC_SIZE;
     read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base);
 
     /*
      * Number required descriptors = number of non-header descriptors +
      *               number of header descriptors +
      *               1x null descriptor
      */
     nhcnt = flexrm_estimate_nonheader_desc_count(msg);
     count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1;
 
     /* Check for available descriptor space. */
     write_offset = ring->bd_write_offset;
     while (count) {
         if (!flexrm_is_next_table_desc(ring->bd_base + write_offset))
             count--;
         write_offset += RING_DESC_SIZE;
         if (write_offset == RING_BD_SIZE)
             write_offset = 0x0;
         if (write_offset == read_offset)
             break;
     }
     if (count) {
         ret = -ENOSPC;
         exit_cleanup = true;
         goto exit;
     }
 
     /* Write descriptors to ring */
     next = flexrm_write_descs(msg, nhcnt, reqid,
             ring->bd_base + ring->bd_write_offset,
             RING_BD_TOGGLE_VALID(ring->bd_write_offset),
             ring->bd_base, ring->bd_base + RING_BD_SIZE);
     if (IS_ERR(next)) {
         ret = PTR_ERR(next);
         exit_cleanup = true;
         goto exit;
     }
 
     /* Save ring BD write offset */
     ring->bd_write_offset = (unsigned long)(next - ring->bd_base);
 
     /* Increment number of messages sent */
     atomic_inc_return(&ring->msg_send_count);
 
 exit:
     /* Update error status in message */
     msg->error = ret;
 
     /* Cleanup if we failed */
     if (exit_cleanup) {
         flexrm_dma_unmap(ring->mbox->dev, msg);
         ring->requests[reqid] = NULL;
         spin_lock_irqsave(&ring->lock, flags);
         bitmap_release_region(ring->requests_bmap, reqid, 0);
         spin_unlock_irqrestore(&ring->lock, flags);
     }
 
     return ret;
 }
 
 static int flexrm_process_completions(struct flexrm_ring *ring)
 {
     u64 desc;
     int err, count = 0;
     unsigned long flags;
     struct brcm_message *msg = NULL;
     u32 reqid, cmpl_read_offset, cmpl_write_offset;
     struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num];
 
     spin_lock_irqsave(&ring->lock, flags);
 
     /*
      * Get current completion read and write offset
      *
      * Note: We should read completion write pointer at least once
      * after we get a MSI interrupt because HW maintains internal
      * MSI status which will allow next MSI interrupt only after
      * completion write pointer is read.
      */
     cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
     cmpl_write_offset *= RING_DESC_SIZE;
     cmpl_read_offset = ring->cmpl_read_offset;
     ring->cmpl_read_offset = cmpl_write_offset;
 
     spin_unlock_irqrestore(&ring->lock, flags);
 
     /* For each completed request notify mailbox clients */
     reqid = 0;
     while (cmpl_read_offset != cmpl_write_offset) {
         /* Dequeue next completion descriptor */
         desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset));
 
         /* Next read offset */
         cmpl_read_offset += RING_DESC_SIZE;
         if (cmpl_read_offset == RING_CMPL_SIZE)
             cmpl_read_offset = 0;
 
         /* Decode error from completion descriptor */
         err = flexrm_cmpl_desc_to_error(desc);
         if (err < 0) {
             dev_warn(ring->mbox->dev,
             "ring%d got completion desc=0x%lx with error %d\n",
             ring->num, (unsigned long)desc, err);
         }
 
         /* Determine request id from completion descriptor */
         reqid = flexrm_cmpl_desc_to_reqid(desc);
 
         /* Determine message pointer based on reqid */
         msg = ring->requests[reqid];
         if (!msg) {
             dev_warn(ring->mbox->dev,
             "ring%d null msg pointer for completion desc=0x%lx\n",
             ring->num, (unsigned long)desc);
             continue;
         }
 
         /* Release reqid for recycling */
         ring->requests[reqid] = NULL;
         spin_lock_irqsave(&ring->lock, flags);
         bitmap_release_region(ring->requests_bmap, reqid, 0);
         spin_unlock_irqrestore(&ring->lock, flags);
 
         /* Unmap DMA mappings */
         flexrm_dma_unmap(ring->mbox->dev, msg);
 
         /* Give-back message to mailbox client */
         msg->error = err;
         mbox_chan_received_data(chan, msg);
 
         /* Increment number of completions processed */
         atomic_inc_return(&ring->msg_cmpl_count);
         count++;
     }
 
     return count;
 }
 
 /* ====== FlexRM Debugfs callbacks ====== */
 
 static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)
 {
     struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
 
     /* Write config in file */
     flexrm_write_config_in_seqfile(mbox, file);
 
     return 0;
 }
 
 static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)
 {
     struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
 
     /* Write stats in file */
     flexrm_write_stats_in_seqfile(mbox, file);
 
     return 0;
 }
 
 /* ====== FlexRM interrupt handler ===== */
 
 static irqreturn_t flexrm_irq_event(int irq, void *dev_id)
 {
     /* We only have MSI for completions so just wakeup IRQ thread */
     /* Ring related errors will be informed via completion descriptors */
 
     return IRQ_WAKE_THREAD;
 }
 
 static irqreturn_t flexrm_irq_thread(int irq, void *dev_id)
 {
     flexrm_process_completions(dev_id);
 
     return IRQ_HANDLED;
 }
 
 /* ====== FlexRM mailbox callbacks ===== */
 
 static int flexrm_send_data(struct mbox_chan *chan, void *data)
 {
     int i, rc;
     struct flexrm_ring *ring = chan->con_priv;
     struct brcm_message *msg = data;
 
     if (msg->type == BRCM_MESSAGE_BATCH) {
         for (i = msg->batch.msgs_queued;
              i < msg->batch.msgs_count; i++) {
             rc = flexrm_new_request(ring, msg,
                          &msg->batch.msgs[i]);
             if (rc) {
                 msg->error = rc;
                 return rc;
             }
             msg->batch.msgs_queued++;
         }
         return 0;
     }
 
     return flexrm_new_request(ring, NULL, data);
 }
 
 static bool flexrm_peek_data(struct mbox_chan *chan)
 {
     int cnt = flexrm_process_completions(chan->con_priv);
 
     return (cnt > 0) ? true : false;
 }
 
 static int flexrm_startup(struct mbox_chan *chan)
 {
     u64 d;
     u32 val, off;
     int ret = 0;
     dma_addr_t next_addr;
     struct flexrm_ring *ring = chan->con_priv;
 
     /* Allocate BD memory */
     ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool,
                        GFP_KERNEL, &ring->bd_dma_base);
     if (!ring->bd_base) {
         dev_err(ring->mbox->dev,
             "can't allocate BD memory for ring%d\n",
             ring->num);
         ret = -ENOMEM;
         goto fail;
     }
 
     /* Configure next table pointer entries in BD memory */
     for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) {
         next_addr = off + RING_DESC_SIZE;
         if (next_addr == RING_BD_SIZE)
             next_addr = 0;
         next_addr += ring->bd_dma_base;
         if (RING_BD_ALIGN_CHECK(next_addr))
             d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off),
                             next_addr);
         else
             d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off));
         flexrm_write_desc(ring->bd_base + off, d);
     }
 
     /* Allocate completion memory */
     ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool,
                      GFP_KERNEL, &ring->cmpl_dma_base);
     if (!ring->cmpl_base) {
         dev_err(ring->mbox->dev,
             "can't allocate completion memory for ring%d\n",
             ring->num);
         ret = -ENOMEM;
         goto fail_free_bd_memory;
     }
 
     /* Request IRQ */
     if (ring->irq == UINT_MAX) {
         dev_err(ring->mbox->dev,
             "ring%d IRQ not available\n", ring->num);
         ret = -ENODEV;
         goto fail_free_cmpl_memory;
     }
     ret = request_threaded_irq(ring->irq,
                    flexrm_irq_event,
                    flexrm_irq_thread,
                    0, dev_name(ring->mbox->dev), ring);
     if (ret) {
         dev_err(ring->mbox->dev,
             "failed to request ring%d IRQ\n", ring->num);
         goto fail_free_cmpl_memory;
     }
     ring->irq_requested = true;
 
     /* Set IRQ affinity hint */
     ring->irq_aff_hint = CPU_MASK_NONE;
     val = ring->mbox->num_rings;
     val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;
     cpumask_set_cpu((ring->num / val) % num_online_cpus(),
             &ring->irq_aff_hint);
     ret = irq_update_affinity_hint(ring->irq, &ring->irq_aff_hint);
     if (ret) {
         dev_err(ring->mbox->dev,
             "failed to set IRQ affinity hint for ring%d\n",
             ring->num);
         goto fail_free_irq;
     }
 
     /* Disable/inactivate ring */
     writel_relaxed(0x0, ring->regs + RING_CONTROL);
 
     /* Program BD start address */
     val = BD_START_ADDR_VALUE(ring->bd_dma_base);
     writel_relaxed(val, ring->regs + RING_BD_START_ADDR);
 
     /* BD write pointer will be same as HW write pointer */
     ring->bd_write_offset =
             readl_relaxed(ring->regs + RING_BD_WRITE_PTR);
     ring->bd_write_offset *= RING_DESC_SIZE;
 
     /* Program completion start address */
     val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);
     writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);
 
     /* Completion read pointer will be same as HW write pointer */
     ring->cmpl_read_offset =
             readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
     ring->cmpl_read_offset *= RING_DESC_SIZE;
 
     /* Read ring Tx, Rx, and Outstanding counts to clear */
     readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS);
     readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS);
     readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS);
     readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS);
     readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND);
 
     /* Configure RING_MSI_CONTROL */
     val = 0;
     val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT);
     val |= BIT(MSI_ENABLE_SHIFT);
     val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
     writel_relaxed(val, ring->regs + RING_MSI_CONTROL);
 
     /* Enable/activate ring */
     val = BIT(CONTROL_ACTIVE_SHIFT);
     writel_relaxed(val, ring->regs + RING_CONTROL);
 
     /* Reset stats to zero */
     atomic_set(&ring->msg_send_count, 0);
     atomic_set(&ring->msg_cmpl_count, 0);
 
     return 0;
 
 fail_free_irq:
     free_irq(ring->irq, ring);
     ring->irq_requested = false;
 fail_free_cmpl_memory:
     dma_pool_free(ring->mbox->cmpl_pool,
               ring->cmpl_base, ring->cmpl_dma_base);
     ring->cmpl_base = NULL;
 fail_free_bd_memory:
     dma_pool_free(ring->mbox->bd_pool,
               ring->bd_base, ring->bd_dma_base);
     ring->bd_base = NULL;
 fail:
     return ret;
 }
 
 static void flexrm_shutdown(struct mbox_chan *chan)
 {
     u32 reqid;
     unsigned int timeout;
     struct brcm_message *msg;
     struct flexrm_ring *ring = chan->con_priv;
 
     /* Disable/inactivate ring */
     writel_relaxed(0x0, ring->regs + RING_CONTROL);
 
     /* Set ring flush state */
     timeout = 1000; /* timeout of 1s */
     writel_relaxed(BIT(CONTROL_FLUSH_SHIFT),
             ring->regs + RING_CONTROL);
     do {
         if (readl_relaxed(ring->regs + RING_FLUSH_DONE) &
             FLUSH_DONE_MASK)
             break;
         mdelay(1);
     } while (--timeout);
     if (!timeout)
         dev_err(ring->mbox->dev,
             "setting ring%d flush state timedout\n", ring->num);
 
     /* Clear ring flush state */
     timeout = 1000; /* timeout of 1s */
     writel_relaxed(0x0, ring->regs + RING_CONTROL);
     do {
         if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) &
               FLUSH_DONE_MASK))
             break;
         mdelay(1);
     } while (--timeout);
     if (!timeout)
         dev_err(ring->mbox->dev,
             "clearing ring%d flush state timedout\n", ring->num);
 
     /* Abort all in-flight requests */
     for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) {
         msg = ring->requests[reqid];
         if (!msg)
             continue;
 
         /* Release reqid for recycling */
         ring->requests[reqid] = NULL;
 
         /* Unmap DMA mappings */
         flexrm_dma_unmap(ring->mbox->dev, msg);
 
         /* Give-back message to mailbox client */
         msg->error = -EIO;
         mbox_chan_received_data(chan, msg);
     }
 
     /* Clear requests bitmap */
     bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
 
     /* Release IRQ */
     if (ring->irq_requested) {
         irq_update_affinity_hint(ring->irq, NULL);
         free_irq(ring->irq, ring);
         ring->irq_requested = false;
     }
 
     /* Free-up completion descriptor ring */
     if (ring->cmpl_base) {
         dma_pool_free(ring->mbox->cmpl_pool,
                   ring->cmpl_base, ring->cmpl_dma_base);
         ring->cmpl_base = NULL;
     }
 
     /* Free-up BD descriptor ring */
     if (ring->bd_base) {
         dma_pool_free(ring->mbox->bd_pool,
                   ring->bd_base, ring->bd_dma_base);
         ring->bd_base = NULL;
     }
 }
 
 static const struct mbox_chan_ops flexrm_mbox_chan_ops = {
     .send_data  = flexrm_send_data,
     .startup    = flexrm_startup,
     .shutdown   = flexrm_shutdown,
     .peek_data  = flexrm_peek_data,
 };
 
 static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr,
                     const struct of_phandle_args *pa)
 {
     struct mbox_chan *chan;
     struct flexrm_ring *ring;
 
     if (pa->args_count < 3)
         return ERR_PTR(-EINVAL);
 
     if (pa->args[0] >= cntlr->num_chans)
         return ERR_PTR(-ENOENT);
 
     if (pa->args[1] > MSI_COUNT_MASK)
         return ERR_PTR(-EINVAL);
 
     if (pa->args[2] > MSI_TIMER_VAL_MASK)
         return ERR_PTR(-EINVAL);
 
     chan = &cntlr->chans[pa->args[0]];
     ring = chan->con_priv;
     ring->msi_count_threshold = pa->args[1];
     ring->msi_timer_val = pa->args[2];
 
     return chan;
 }
 
 /* ====== FlexRM platform driver ===== */
 
 static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg)
 {
     struct device *dev = msi_desc_to_dev(desc);
     struct flexrm_mbox *mbox = dev_get_drvdata(dev);
     struct flexrm_ring *ring = &mbox->rings[desc->msi_index];
 
     /* Configure per-Ring MSI registers */
     writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS);
     writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS);
     writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE);
 }
 
 static int flexrm_mbox_probe(struct platform_device *pdev)
 {
     int index, ret = 0;
     void __iomem *regs;
     void __iomem *regs_end;
     struct resource *iomem;
     struct flexrm_ring *ring;
     struct flexrm_mbox *mbox;
     struct device *dev = &pdev->dev;
 
     /* Allocate driver mailbox struct */
     mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
     if (!mbox) {
         ret = -ENOMEM;
         goto fail;
     }
     mbox->dev = dev;
     platform_set_drvdata(pdev, mbox);
 
     /* Get resource for registers */
     iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
     if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) {
         ret = -ENODEV;
         goto fail;
     }
 
     /* Map registers of all rings */
     mbox->regs = devm_ioremap_resource(&pdev->dev, iomem);
     if (IS_ERR(mbox->regs)) {
         ret = PTR_ERR(mbox->regs);
         goto fail;
     }
     regs_end = mbox->regs + resource_size(iomem);
 
     /* Scan and count available rings */
     mbox->num_rings = 0;
     for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) {
         if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC)
             mbox->num_rings++;
     }
     if (!mbox->num_rings) {
         ret = -ENODEV;
         goto fail;
     }
 
     /* Allocate driver ring structs */
     ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL);
     if (!ring) {
         ret = -ENOMEM;
         goto fail;
     }
     mbox->rings = ring;
 
     /* Initialize members of driver ring structs */
     regs = mbox->regs;
     for (index = 0; index < mbox->num_rings; index++) {
         ring = &mbox->rings[index];
         ring->num = index;
         ring->mbox = mbox;
         while ((regs < regs_end) &&
                (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC))
             regs += RING_REGS_SIZE;
         if (regs_end <= regs) {
             ret = -ENODEV;
             goto fail;
         }
         ring->regs = regs;
         regs += RING_REGS_SIZE;
         ring->irq = UINT_MAX;
         ring->irq_requested = false;
         ring->msi_timer_val = MSI_TIMER_VAL_MASK;
         ring->msi_count_threshold = 0x1;
         memset(ring->requests, 0, sizeof(ring->requests));
         ring->bd_base = NULL;
         ring->bd_dma_base = 0;
         ring->cmpl_base = NULL;
         ring->cmpl_dma_base = 0;
         atomic_set(&ring->msg_send_count, 0);
         atomic_set(&ring->msg_cmpl_count, 0);
         spin_lock_init(&ring->lock);
         bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
         ring->cmpl_read_offset = 0;
     }
 
     /* FlexRM is capable of 40-bit physical addresses only */
     ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
     if (ret) {
         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
         if (ret)
             goto fail;
     }
 
     /* Create DMA pool for ring BD memory */
     mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE,
                     1 << RING_BD_ALIGN_ORDER, 0);
     if (!mbox->bd_pool) {
         ret = -ENOMEM;
         goto fail;
     }
 
     /* Create DMA pool for ring completion memory */
     mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE,
                       1 << RING_CMPL_ALIGN_ORDER, 0);
     if (!mbox->cmpl_pool) {
         ret = -ENOMEM;
         goto fail_destroy_bd_pool;
     }
 
     /* Allocate platform MSIs for each ring */
     ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings,
                         flexrm_mbox_msi_write);
     if (ret)
         goto fail_destroy_cmpl_pool;
 
     /* Save alloced IRQ numbers for each ring */
     for (index = 0; index < mbox->num_rings; index++)
         mbox->rings[index].irq = msi_get_virq(dev, index);
 
     /* Check availability of debugfs */
     if (!debugfs_initialized())
         goto skip_debugfs;
 
     /* Create debugfs root entry */
     mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);
 
     /* Create debugfs config entry */
     debugfs_create_devm_seqfile(mbox->dev, "config", mbox->root,
                     flexrm_debugfs_conf_show);
 
     /* Create debugfs stats entry */
     debugfs_create_devm_seqfile(mbox->dev, "stats", mbox->root,
                     flexrm_debugfs_stats_show);
 
 skip_debugfs:
 
     /* Initialize mailbox controller */
     mbox->controller.txdone_irq = false;
     mbox->controller.txdone_poll = false;
     mbox->controller.ops = &flexrm_mbox_chan_ops;
     mbox->controller.dev = dev;
     mbox->controller.num_chans = mbox->num_rings;
     mbox->controller.of_xlate = flexrm_mbox_of_xlate;
     mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings,
                 sizeof(*mbox->controller.chans), GFP_KERNEL);
     if (!mbox->controller.chans) {
         ret = -ENOMEM;
         goto fail_free_debugfs_root;
     }
     for (index = 0; index < mbox->num_rings; index++)
         mbox->controller.chans[index].con_priv = &mbox->rings[index];
 
     /* Register mailbox controller */
     ret = devm_mbox_controller_register(dev, &mbox->controller);
     if (ret)
         goto fail_free_debugfs_root;
 
     dev_info(dev, "registered flexrm mailbox with %d channels\n",
             mbox->controller.num_chans);
 
     return 0;
 
 fail_free_debugfs_root:
     debugfs_remove_recursive(mbox->root);
     platform_msi_domain_free_irqs(dev);
 fail_destroy_cmpl_pool:
     dma_pool_destroy(mbox->cmpl_pool);
 fail_destroy_bd_pool:
     dma_pool_destroy(mbox->bd_pool);
 fail:
     return ret;
 }
 
 static int flexrm_mbox_remove(struct platform_device *pdev)
 {
     struct device *dev = &pdev->dev;
     struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
 
     debugfs_remove_recursive(mbox->root);
 
     platform_msi_domain_free_irqs(dev);
 
     dma_pool_destroy(mbox->cmpl_pool);
     dma_pool_destroy(mbox->bd_pool);
 
     return 0;
 }
 
 static const struct of_device_id flexrm_mbox_of_match[] = {
     { .compatible = "brcm,iproc-flexrm-mbox", },
     {},
 };
 MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match);
 
 static struct platform_driver flexrm_mbox_driver = {
     .driver = {
         .name = "brcm-flexrm-mbox",
         .of_match_table = flexrm_mbox_of_match,
     },
     .probe      = flexrm_mbox_probe,
     .remove     = flexrm_mbox_remove,
 };
 module_platform_driver(flexrm_mbox_driver);
 
 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
 MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver");
 MODULE_LICENSE("GPL v2");

This page was automatically generated by the 2.1.0 LXR engine. The LXR team