OSCL-LXR linux-6.0.1/​drivers/​dma/​lgm/​lgm-dma.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Lightning Mountain centralized DMA controller driver
  *
  * Copyright (c) 2016 - 2020 Intel Corporation.
  */
 
 #include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmapool.h>
 #include <linux/err.h>
 #include <linux/export.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/iopoll.h>
 #include <linux/of_dma.h>
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 #include <linux/reset.h>
 
 #include "../dmaengine.h"
 #include "../virt-dma.h"
 
 #define DRIVER_NAME         "lgm-dma"
 
 #define DMA_ID              0x0008
 #define DMA_ID_REV          GENMASK(7, 0)
 #define DMA_ID_PNR          GENMASK(19, 16)
 #define DMA_ID_CHNR         GENMASK(26, 20)
 #define DMA_ID_DW_128B          BIT(27)
 #define DMA_ID_AW_36B           BIT(28)
 #define DMA_VER32           0x32
 #define DMA_VER31           0x31
 #define DMA_VER22           0x0A
 
 #define DMA_CTRL            0x0010
 #define DMA_CTRL_RST            BIT(0)
 #define DMA_CTRL_DSRAM_PATH     BIT(1)
 #define DMA_CTRL_DBURST_WR      BIT(3)
 #define DMA_CTRL_VLD_DF_ACK     BIT(4)
 #define DMA_CTRL_CH_FL          BIT(6)
 #define DMA_CTRL_DS_FOD         BIT(7)
 #define DMA_CTRL_DRB            BIT(8)
 #define DMA_CTRL_ENBE           BIT(9)
 #define DMA_CTRL_DESC_TMOUT_CNT_V31 GENMASK(27, 16)
 #define DMA_CTRL_DESC_TMOUT_EN_V31  BIT(30)
 #define DMA_CTRL_PKTARB         BIT(31)
 
 #define DMA_CPOLL           0x0014
 #define DMA_CPOLL_CNT           GENMASK(15, 4)
 #define DMA_CPOLL_EN            BIT(31)
 
 #define DMA_CS              0x0018
 #define DMA_CS_MASK         GENMASK(5, 0)
 
 #define DMA_CCTRL           0x001C
 #define DMA_CCTRL_ON            BIT(0)
 #define DMA_CCTRL_RST           BIT(1)
 #define DMA_CCTRL_CH_POLL_EN        BIT(2)
 #define DMA_CCTRL_CH_ABC        BIT(3) /* Adaptive Burst Chop */
 #define DMA_CDBA_MSB            GENMASK(7, 4)
 #define DMA_CCTRL_DIR_TX        BIT(8)
 #define DMA_CCTRL_CLASS         GENMASK(11, 9)
 #define DMA_CCTRL_CLASSH        GENMASK(19, 18)
 #define DMA_CCTRL_WR_NP_EN      BIT(21)
 #define DMA_CCTRL_PDEN          BIT(23)
 #define DMA_MAX_CLASS           (SZ_32 - 1)
 
 #define DMA_CDBA            0x0020
 #define DMA_CDLEN           0x0024
 #define DMA_CIS             0x0028
 #define DMA_CIE             0x002C
 #define DMA_CI_EOP          BIT(1)
 #define DMA_CI_DUR          BIT(2)
 #define DMA_CI_DESCPT           BIT(3)
 #define DMA_CI_CHOFF            BIT(4)
 #define DMA_CI_RDERR            BIT(5)
 #define DMA_CI_ALL                          \
     (DMA_CI_EOP | DMA_CI_DUR | DMA_CI_DESCPT | DMA_CI_CHOFF | DMA_CI_RDERR)
 
 #define DMA_PS              0x0040
 #define DMA_PCTRL           0x0044
 #define DMA_PCTRL_RXBL16        BIT(0)
 #define DMA_PCTRL_TXBL16        BIT(1)
 #define DMA_PCTRL_RXBL          GENMASK(3, 2)
 #define DMA_PCTRL_RXBL_8        3
 #define DMA_PCTRL_TXBL          GENMASK(5, 4)
 #define DMA_PCTRL_TXBL_8        3
 #define DMA_PCTRL_PDEN          BIT(6)
 #define DMA_PCTRL_RXBL32        BIT(7)
 #define DMA_PCTRL_RXENDI        GENMASK(9, 8)
 #define DMA_PCTRL_TXENDI        GENMASK(11, 10)
 #define DMA_PCTRL_TXBL32        BIT(15)
 #define DMA_PCTRL_MEM_FLUSH     BIT(16)
 
 #define DMA_IRNEN1          0x00E8
 #define DMA_IRNCR1          0x00EC
 #define DMA_IRNEN           0x00F4
 #define DMA_IRNCR           0x00F8
 #define DMA_C_DP_TICK           0x100
 #define DMA_C_DP_TICK_TIKNARB       GENMASK(15, 0)
 #define DMA_C_DP_TICK_TIKARB        GENMASK(31, 16)
 
 #define DMA_C_HDRM          0x110
 /*
  * If header mode is set in DMA descriptor,
  *   If bit 30 is disabled, HDR_LEN must be configured according to channel
  *     requirement.
  *   If bit 30 is enabled(checksum with heade mode), HDR_LEN has no need to
  *     be configured. It will enable check sum for switch
  * If header mode is not set in DMA descriptor,
  *   This register setting doesn't matter
  */
 #define DMA_C_HDRM_HDR_SUM      BIT(30)
 
 #define DMA_C_BOFF          0x120
 #define DMA_C_BOFF_BOF_LEN      GENMASK(7, 0)
 #define DMA_C_BOFF_EN           BIT(31)
 
 #define DMA_ORRC            0x190
 #define DMA_ORRC_ORRCNT         GENMASK(8, 4)
 #define DMA_ORRC_EN         BIT(31)
 
 #define DMA_C_ENDIAN            0x200
 #define DMA_C_END_DATAENDI      GENMASK(1, 0)
 #define DMA_C_END_DE_EN         BIT(7)
 #define DMA_C_END_DESENDI       GENMASK(9, 8)
 #define DMA_C_END_DES_EN        BIT(16)
 
 /* DMA controller capability */
 #define DMA_ADDR_36BIT          BIT(0)
 #define DMA_DATA_128BIT         BIT(1)
 #define DMA_CHAN_FLOW_CTL       BIT(2)
 #define DMA_DESC_FOD            BIT(3)
 #define DMA_DESC_IN_SRAM        BIT(4)
 #define DMA_EN_BYTE_EN          BIT(5)
 #define DMA_DBURST_WR           BIT(6)
 #define DMA_VALID_DESC_FETCH_ACK    BIT(7)
 #define DMA_DFT_DRB         BIT(8)
 
 #define DMA_ORRC_MAX_CNT        (SZ_32 - 1)
 #define DMA_DFT_POLL_CNT        SZ_4
 #define DMA_DFT_BURST_V22       SZ_2
 #define DMA_BURSTL_8DW          SZ_8
 #define DMA_BURSTL_16DW         SZ_16
 #define DMA_BURSTL_32DW         SZ_32
 #define DMA_DFT_BURST           DMA_BURSTL_16DW
 #define DMA_MAX_DESC_NUM        (SZ_8K - 1)
 #define DMA_CHAN_BOFF_MAX       (SZ_256 - 1)
 #define DMA_DFT_ENDIAN          0
 
 #define DMA_DFT_DESC_TCNT       50
 #define DMA_HDR_LEN_MAX         (SZ_16K - 1)
 
 /* DMA flags */
 #define DMA_TX_CH           BIT(0)
 #define DMA_RX_CH           BIT(1)
 #define DEVICE_ALLOC_DESC       BIT(2)
 #define CHAN_IN_USE         BIT(3)
 #define DMA_HW_DESC         BIT(4)
 
 /* Descriptor fields */
 #define DESC_DATA_LEN           GENMASK(15, 0)
 #define DESC_BYTE_OFF           GENMASK(25, 23)
 #define DESC_EOP            BIT(28)
 #define DESC_SOP            BIT(29)
 #define DESC_C              BIT(30)
 #define DESC_OWN            BIT(31)
 
 #define DMA_CHAN_RST            1
 #define DMA_MAX_SIZE            (BIT(16) - 1)
 #define MAX_LOWER_CHANS         32
 #define MASK_LOWER_CHANS        GENMASK(4, 0)
 #define DMA_OWN             1
 #define HIGH_4_BITS         GENMASK(3, 0)
 #define DMA_DFT_DESC_NUM        1
 #define DMA_PKT_DROP_DIS        0
 
 enum ldma_chan_on_off {
     DMA_CH_OFF = 0,
     DMA_CH_ON = 1,
 };
 
 enum {
     DMA_TYPE_TX = 0,
     DMA_TYPE_RX,
     DMA_TYPE_MCPY,
 };
 
 struct ldma_dev;
 struct ldma_port;
 
 struct ldma_chan {
     struct virt_dma_chan    vchan;
     struct ldma_port    *port; /* back pointer */
     char            name[8]; /* Channel name */
     int         nr; /* Channel id in hardware */
     u32         flags; /* central way or channel based way */
     enum ldma_chan_on_off   onoff;
     dma_addr_t      desc_phys;
     void            *desc_base; /* Virtual address */
     u32         desc_cnt; /* Number of descriptors */
     int         rst;
     u32         hdrm_len;
     bool            hdrm_csum;
     u32         boff_len;
     u32         data_endian;
     u32         desc_endian;
     bool            pden;
     bool            desc_rx_np;
     bool            data_endian_en;
     bool            desc_endian_en;
     bool            abc_en;
     bool            desc_init;
     struct dma_pool     *desc_pool; /* Descriptors pool */
     u32         desc_num;
     struct dw2_desc_sw  *ds;
     struct work_struct  work;
     struct dma_slave_config config;
 };
 
 struct ldma_port {
     struct ldma_dev     *ldev; /* back pointer */
     u32         portid;
     u32         rxbl;
     u32         txbl;
     u32         rxendi;
     u32         txendi;
     u32         pkt_drop;
 };
 
 /* Instance specific data */
 struct ldma_inst_data {
     bool            desc_in_sram;
     bool            chan_fc;
     bool            desc_fod; /* Fetch On Demand */
     bool            valid_desc_fetch_ack;
     u32         orrc; /* Outstanding read count */
     const char      *name;
     u32         type;
 };
 
 struct ldma_dev {
     struct device       *dev;
     void __iomem        *base;
     struct reset_control    *rst;
     struct clk      *core_clk;
     struct dma_device   dma_dev;
     u32         ver;
     int         irq;
     struct ldma_port    *ports;
     struct ldma_chan    *chans; /* channel list on this DMA or port */
     spinlock_t      dev_lock; /* Controller register exclusive */
     u32         chan_nrs;
     u32         port_nrs;
     u32         channels_mask;
     u32         flags;
     u32         pollcnt;
     const struct ldma_inst_data *inst;
     struct workqueue_struct *wq;
 };
 
 struct dw2_desc {
     u32 field;
     u32 addr;
 } __packed __aligned(8);
 
 struct dw2_desc_sw {
     struct virt_dma_desc    vdesc;
     struct ldma_chan    *chan;
     dma_addr_t      desc_phys;
     size_t          desc_cnt;
     size_t          size;
     struct dw2_desc     *desc_hw;
 };
 
 static inline void
 ldma_update_bits(struct ldma_dev *d, u32 mask, u32 val, u32 ofs)
 {
     u32 old_val, new_val;
 
     old_val = readl(d->base +  ofs);
     new_val = (old_val & ~mask) | (val & mask);
 
     if (new_val != old_val)
         writel(new_val, d->base + ofs);
 }
 
 static inline struct ldma_chan *to_ldma_chan(struct dma_chan *chan)
 {
     return container_of(chan, struct ldma_chan, vchan.chan);
 }
 
 static inline struct ldma_dev *to_ldma_dev(struct dma_device *dma_dev)
 {
     return container_of(dma_dev, struct ldma_dev, dma_dev);
 }
 
 static inline struct dw2_desc_sw *to_lgm_dma_desc(struct virt_dma_desc *vdesc)
 {
     return container_of(vdesc, struct dw2_desc_sw, vdesc);
 }
 
 static inline bool ldma_chan_tx(struct ldma_chan *c)
 {
     return !!(c->flags & DMA_TX_CH);
 }
 
 static inline bool ldma_chan_is_hw_desc(struct ldma_chan *c)
 {
     return !!(c->flags & DMA_HW_DESC);
 }
 
 static void ldma_dev_reset(struct ldma_dev *d)
 
 {
     unsigned long flags;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, DMA_CTRL_RST, DMA_CTRL_RST, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_pkt_arb_cfg(struct ldma_dev *d, bool enable)
 {
     unsigned long flags;
     u32 mask = DMA_CTRL_PKTARB;
     u32 val = enable ? DMA_CTRL_PKTARB : 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_sram_desc_cfg(struct ldma_dev *d, bool enable)
 {
     unsigned long flags;
     u32 mask = DMA_CTRL_DSRAM_PATH;
     u32 val = enable ? DMA_CTRL_DSRAM_PATH : 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_chan_flow_ctl_cfg(struct ldma_dev *d, bool enable)
 {
     unsigned long flags;
     u32 mask, val;
 
     if (d->inst->type != DMA_TYPE_TX)
         return;
 
     mask = DMA_CTRL_CH_FL;
     val = enable ? DMA_CTRL_CH_FL : 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_global_polling_enable(struct ldma_dev *d)
 {
     unsigned long flags;
     u32 mask = DMA_CPOLL_EN | DMA_CPOLL_CNT;
     u32 val = DMA_CPOLL_EN;
 
     val |= FIELD_PREP(DMA_CPOLL_CNT, d->pollcnt);
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CPOLL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_desc_fetch_on_demand_cfg(struct ldma_dev *d, bool enable)
 {
     unsigned long flags;
     u32 mask, val;
 
     if (d->inst->type == DMA_TYPE_MCPY)
         return;
 
     mask = DMA_CTRL_DS_FOD;
     val = enable ? DMA_CTRL_DS_FOD : 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_byte_enable_cfg(struct ldma_dev *d, bool enable)
 {
     unsigned long flags;
     u32 mask = DMA_CTRL_ENBE;
     u32 val = enable ? DMA_CTRL_ENBE : 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_orrc_cfg(struct ldma_dev *d)
 {
     unsigned long flags;
     u32 val = 0;
     u32 mask;
 
     if (d->inst->type == DMA_TYPE_RX)
         return;
 
     mask = DMA_ORRC_EN | DMA_ORRC_ORRCNT;
     if (d->inst->orrc > 0 && d->inst->orrc <= DMA_ORRC_MAX_CNT)
         val = DMA_ORRC_EN | FIELD_PREP(DMA_ORRC_ORRCNT, d->inst->orrc);
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_ORRC);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_df_tout_cfg(struct ldma_dev *d, bool enable, int tcnt)
 {
     u32 mask = DMA_CTRL_DESC_TMOUT_CNT_V31;
     unsigned long flags;
     u32 val;
 
     if (enable)
         val = DMA_CTRL_DESC_TMOUT_EN_V31 | FIELD_PREP(DMA_CTRL_DESC_TMOUT_CNT_V31, tcnt);
     else
         val = 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_dburst_wr_cfg(struct ldma_dev *d, bool enable)
 {
     unsigned long flags;
     u32 mask, val;
 
     if (d->inst->type != DMA_TYPE_RX && d->inst->type != DMA_TYPE_MCPY)
         return;
 
     mask = DMA_CTRL_DBURST_WR;
     val = enable ? DMA_CTRL_DBURST_WR : 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_vld_fetch_ack_cfg(struct ldma_dev *d, bool enable)
 {
     unsigned long flags;
     u32 mask, val;
 
     if (d->inst->type != DMA_TYPE_TX)
         return;
 
     mask = DMA_CTRL_VLD_DF_ACK;
     val = enable ? DMA_CTRL_VLD_DF_ACK : 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_dev_drb_cfg(struct ldma_dev *d, int enable)
 {
     unsigned long flags;
     u32 mask = DMA_CTRL_DRB;
     u32 val = enable ? DMA_CTRL_DRB : 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, mask, val, DMA_CTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static int ldma_dev_cfg(struct ldma_dev *d)
 {
     bool enable;
 
     ldma_dev_pkt_arb_cfg(d, true);
     ldma_dev_global_polling_enable(d);
 
     enable = !!(d->flags & DMA_DFT_DRB);
     ldma_dev_drb_cfg(d, enable);
 
     enable = !!(d->flags & DMA_EN_BYTE_EN);
     ldma_dev_byte_enable_cfg(d, enable);
 
     enable = !!(d->flags & DMA_CHAN_FLOW_CTL);
     ldma_dev_chan_flow_ctl_cfg(d, enable);
 
     enable = !!(d->flags & DMA_DESC_FOD);
     ldma_dev_desc_fetch_on_demand_cfg(d, enable);
 
     enable = !!(d->flags & DMA_DESC_IN_SRAM);
     ldma_dev_sram_desc_cfg(d, enable);
 
     enable = !!(d->flags & DMA_DBURST_WR);
     ldma_dev_dburst_wr_cfg(d, enable);
 
     enable = !!(d->flags & DMA_VALID_DESC_FETCH_ACK);
     ldma_dev_vld_fetch_ack_cfg(d, enable);
 
     if (d->ver > DMA_VER22) {
         ldma_dev_orrc_cfg(d);
         ldma_dev_df_tout_cfg(d, true, DMA_DFT_DESC_TCNT);
     }
 
     dev_dbg(d->dev, "%s Controller 0x%08x configuration done\n",
         d->inst->name, readl(d->base + DMA_CTRL));
 
     return 0;
 }
 
 static int ldma_chan_cctrl_cfg(struct ldma_chan *c, u32 val)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     u32 class_low, class_high;
     unsigned long flags;
     u32 reg;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     reg = readl(d->base + DMA_CCTRL);
     /* Read from hardware */
     if (reg & DMA_CCTRL_DIR_TX)
         c->flags |= DMA_TX_CH;
     else
         c->flags |= DMA_RX_CH;
 
     /* Keep the class value unchanged */
     class_low = FIELD_GET(DMA_CCTRL_CLASS, reg);
     class_high = FIELD_GET(DMA_CCTRL_CLASSH, reg);
     val &= ~DMA_CCTRL_CLASS;
     val |= FIELD_PREP(DMA_CCTRL_CLASS, class_low);
     val &= ~DMA_CCTRL_CLASSH;
     val |= FIELD_PREP(DMA_CCTRL_CLASSH, class_high);
     writel(val, d->base + DMA_CCTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 
     return 0;
 }
 
 static void ldma_chan_irq_init(struct ldma_chan *c)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     unsigned long flags;
     u32 enofs, crofs;
     u32 cn_bit;
 
     if (c->nr < MAX_LOWER_CHANS) {
         enofs = DMA_IRNEN;
         crofs = DMA_IRNCR;
     } else {
         enofs = DMA_IRNEN1;
         crofs = DMA_IRNCR1;
     }
 
     cn_bit = BIT(c->nr & MASK_LOWER_CHANS);
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
 
     /* Clear all interrupts and disabled it */
     writel(0, d->base + DMA_CIE);
     writel(DMA_CI_ALL, d->base + DMA_CIS);
 
     ldma_update_bits(d, cn_bit, 0, enofs);
     writel(cn_bit, d->base + crofs);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_chan_set_class(struct ldma_chan *c, u32 val)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     u32 class_val;
 
     if (d->inst->type == DMA_TYPE_MCPY || val > DMA_MAX_CLASS)
         return;
 
     /* 3 bits low */
     class_val = FIELD_PREP(DMA_CCTRL_CLASS, val & 0x7);
     /* 2 bits high */
     class_val |= FIELD_PREP(DMA_CCTRL_CLASSH, (val >> 3) & 0x3);
 
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, DMA_CCTRL_CLASS | DMA_CCTRL_CLASSH, class_val,
              DMA_CCTRL);
 }
 
 static int ldma_chan_on(struct ldma_chan *c)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     unsigned long flags;
 
     /* If descriptors not configured, not allow to turn on channel */
     if (WARN_ON(!c->desc_init))
         return -EINVAL;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, DMA_CCTRL_ON, DMA_CCTRL_ON, DMA_CCTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 
     c->onoff = DMA_CH_ON;
 
     return 0;
 }
 
 static int ldma_chan_off(struct ldma_chan *c)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     unsigned long flags;
     u32 val;
     int ret;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, DMA_CCTRL_ON, 0, DMA_CCTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 
     ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val,
                     !(val & DMA_CCTRL_ON), 0, 10000);
     if (ret)
         return ret;
 
     c->onoff = DMA_CH_OFF;
 
     return 0;
 }
 
 static void ldma_chan_desc_hw_cfg(struct ldma_chan *c, dma_addr_t desc_base,
                   int desc_num)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     unsigned long flags;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     writel(lower_32_bits(desc_base), d->base + DMA_CDBA);
 
     /* Higher 4 bits of 36 bit addressing */
     if (IS_ENABLED(CONFIG_64BIT)) {
         u32 hi = upper_32_bits(desc_base) & HIGH_4_BITS;
 
         ldma_update_bits(d, DMA_CDBA_MSB,
                  FIELD_PREP(DMA_CDBA_MSB, hi), DMA_CCTRL);
     }
     writel(desc_num, d->base + DMA_CDLEN);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 
     c->desc_init = true;
 }
 
 static struct dma_async_tx_descriptor *
 ldma_chan_desc_cfg(struct dma_chan *chan, dma_addr_t desc_base, int desc_num)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     struct dma_async_tx_descriptor *tx;
     struct dw2_desc_sw *ds;
 
     if (!desc_num) {
         dev_err(d->dev, "Channel %d must allocate descriptor first\n",
             c->nr);
         return NULL;
     }
 
     if (desc_num > DMA_MAX_DESC_NUM) {
         dev_err(d->dev, "Channel %d descriptor number out of range %d\n",
             c->nr, desc_num);
         return NULL;
     }
 
     ldma_chan_desc_hw_cfg(c, desc_base, desc_num);
 
     c->flags |= DMA_HW_DESC;
     c->desc_cnt = desc_num;
     c->desc_phys = desc_base;
 
     ds = kzalloc(sizeof(*ds), GFP_NOWAIT);
     if (!ds)
         return NULL;
 
     tx = &ds->vdesc.tx;
     dma_async_tx_descriptor_init(tx, chan);
 
     return tx;
 }
 
 static int ldma_chan_reset(struct ldma_chan *c)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     unsigned long flags;
     u32 val;
     int ret;
 
     ret = ldma_chan_off(c);
     if (ret)
         return ret;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, DMA_CCTRL_RST, DMA_CCTRL_RST, DMA_CCTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 
     ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val,
                     !(val & DMA_CCTRL_RST), 0, 10000);
     if (ret)
         return ret;
 
     c->rst = 1;
     c->desc_init = false;
 
     return 0;
 }
 
 static void ldma_chan_byte_offset_cfg(struct ldma_chan *c, u32 boff_len)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     u32 mask = DMA_C_BOFF_EN | DMA_C_BOFF_BOF_LEN;
     u32 val;
 
     if (boff_len > 0 && boff_len <= DMA_CHAN_BOFF_MAX)
         val = FIELD_PREP(DMA_C_BOFF_BOF_LEN, boff_len) | DMA_C_BOFF_EN;
     else
         val = 0;
 
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, mask, val, DMA_C_BOFF);
 }
 
 static void ldma_chan_data_endian_cfg(struct ldma_chan *c, bool enable,
                       u32 endian_type)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     u32 mask = DMA_C_END_DE_EN | DMA_C_END_DATAENDI;
     u32 val;
 
     if (enable)
         val = DMA_C_END_DE_EN | FIELD_PREP(DMA_C_END_DATAENDI, endian_type);
     else
         val = 0;
 
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, mask, val, DMA_C_ENDIAN);
 }
 
 static void ldma_chan_desc_endian_cfg(struct ldma_chan *c, bool enable,
                       u32 endian_type)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     u32 mask = DMA_C_END_DES_EN | DMA_C_END_DESENDI;
     u32 val;
 
     if (enable)
         val = DMA_C_END_DES_EN | FIELD_PREP(DMA_C_END_DESENDI, endian_type);
     else
         val = 0;
 
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, mask, val, DMA_C_ENDIAN);
 }
 
 static void ldma_chan_hdr_mode_cfg(struct ldma_chan *c, u32 hdr_len, bool csum)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     u32 mask, val;
 
     /* NB, csum disabled, hdr length must be provided */
     if (!csum && (!hdr_len || hdr_len > DMA_HDR_LEN_MAX))
         return;
 
     mask = DMA_C_HDRM_HDR_SUM;
     val = DMA_C_HDRM_HDR_SUM;
 
     if (!csum && hdr_len)
         val = hdr_len;
 
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, mask, val, DMA_C_HDRM);
 }
 
 static void ldma_chan_rxwr_np_cfg(struct ldma_chan *c, bool enable)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     u32 mask, val;
 
     /* Only valid for RX channel */
     if (ldma_chan_tx(c))
         return;
 
     mask = DMA_CCTRL_WR_NP_EN;
     val = enable ? DMA_CCTRL_WR_NP_EN : 0;
 
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, mask, val, DMA_CCTRL);
 }
 
 static void ldma_chan_abc_cfg(struct ldma_chan *c, bool enable)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     u32 mask, val;
 
     if (d->ver < DMA_VER32 || ldma_chan_tx(c))
         return;
 
     mask = DMA_CCTRL_CH_ABC;
     val = enable ? DMA_CCTRL_CH_ABC : 0;
 
     ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS);
     ldma_update_bits(d, mask, val, DMA_CCTRL);
 }
 
 static int ldma_port_cfg(struct ldma_port *p)
 {
     unsigned long flags;
     struct ldma_dev *d;
     u32 reg;
 
     d = p->ldev;
     reg = FIELD_PREP(DMA_PCTRL_TXENDI, p->txendi);
     reg |= FIELD_PREP(DMA_PCTRL_RXENDI, p->rxendi);
 
     if (d->ver == DMA_VER22) {
         reg |= FIELD_PREP(DMA_PCTRL_TXBL, p->txbl);
         reg |= FIELD_PREP(DMA_PCTRL_RXBL, p->rxbl);
     } else {
         reg |= FIELD_PREP(DMA_PCTRL_PDEN, p->pkt_drop);
 
         if (p->txbl == DMA_BURSTL_32DW)
             reg |= DMA_PCTRL_TXBL32;
         else if (p->txbl == DMA_BURSTL_16DW)
             reg |= DMA_PCTRL_TXBL16;
         else
             reg |= FIELD_PREP(DMA_PCTRL_TXBL, DMA_PCTRL_TXBL_8);
 
         if (p->rxbl == DMA_BURSTL_32DW)
             reg |= DMA_PCTRL_RXBL32;
         else if (p->rxbl == DMA_BURSTL_16DW)
             reg |= DMA_PCTRL_RXBL16;
         else
             reg |= FIELD_PREP(DMA_PCTRL_RXBL, DMA_PCTRL_RXBL_8);
     }
 
     spin_lock_irqsave(&d->dev_lock, flags);
     writel(p->portid, d->base + DMA_PS);
     writel(reg, d->base + DMA_PCTRL);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 
     reg = readl(d->base + DMA_PCTRL); /* read back */
     dev_dbg(d->dev, "Port Control 0x%08x configuration done\n", reg);
 
     return 0;
 }
 
 static int ldma_chan_cfg(struct ldma_chan *c)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     unsigned long flags;
     u32 reg;
 
     reg = c->pden ? DMA_CCTRL_PDEN : 0;
     reg |= c->onoff ? DMA_CCTRL_ON : 0;
     reg |= c->rst ? DMA_CCTRL_RST : 0;
 
     ldma_chan_cctrl_cfg(c, reg);
     ldma_chan_irq_init(c);
 
     if (d->ver <= DMA_VER22)
         return 0;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     ldma_chan_set_class(c, c->nr);
     ldma_chan_byte_offset_cfg(c, c->boff_len);
     ldma_chan_data_endian_cfg(c, c->data_endian_en, c->data_endian);
     ldma_chan_desc_endian_cfg(c, c->desc_endian_en, c->desc_endian);
     ldma_chan_hdr_mode_cfg(c, c->hdrm_len, c->hdrm_csum);
     ldma_chan_rxwr_np_cfg(c, c->desc_rx_np);
     ldma_chan_abc_cfg(c, c->abc_en);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 
     if (ldma_chan_is_hw_desc(c))
         ldma_chan_desc_hw_cfg(c, c->desc_phys, c->desc_cnt);
 
     return 0;
 }
 
 static void ldma_dev_init(struct ldma_dev *d)
 {
     unsigned long ch_mask = (unsigned long)d->channels_mask;
     struct ldma_port *p;
     struct ldma_chan *c;
     int i;
     u32 j;
 
     spin_lock_init(&d->dev_lock);
     ldma_dev_reset(d);
     ldma_dev_cfg(d);
 
     /* DMA port initialization */
     for (i = 0; i < d->port_nrs; i++) {
         p = &d->ports[i];
         ldma_port_cfg(p);
     }
 
     /* DMA channel initialization */
     for_each_set_bit(j, &ch_mask, d->chan_nrs) {
         c = &d->chans[j];
         ldma_chan_cfg(c);
     }
 }
 
 static int ldma_cfg_init(struct ldma_dev *d)
 {
     struct fwnode_handle *fwnode = dev_fwnode(d->dev);
     struct ldma_port *p;
     int i;
 
     if (fwnode_property_read_bool(fwnode, "intel,dma-byte-en"))
         d->flags |= DMA_EN_BYTE_EN;
 
     if (fwnode_property_read_bool(fwnode, "intel,dma-dburst-wr"))
         d->flags |= DMA_DBURST_WR;
 
     if (fwnode_property_read_bool(fwnode, "intel,dma-drb"))
         d->flags |= DMA_DFT_DRB;
 
     if (fwnode_property_read_u32(fwnode, "intel,dma-poll-cnt",
                      &d->pollcnt))
         d->pollcnt = DMA_DFT_POLL_CNT;
 
     if (d->inst->chan_fc)
         d->flags |= DMA_CHAN_FLOW_CTL;
 
     if (d->inst->desc_fod)
         d->flags |= DMA_DESC_FOD;
 
     if (d->inst->desc_in_sram)
         d->flags |= DMA_DESC_IN_SRAM;
 
     if (d->inst->valid_desc_fetch_ack)
         d->flags |= DMA_VALID_DESC_FETCH_ACK;
 
     if (d->ver > DMA_VER22) {
         if (!d->port_nrs)
             return -EINVAL;
 
         for (i = 0; i < d->port_nrs; i++) {
             p = &d->ports[i];
             p->rxendi = DMA_DFT_ENDIAN;
             p->txendi = DMA_DFT_ENDIAN;
             p->rxbl = DMA_DFT_BURST;
             p->txbl = DMA_DFT_BURST;
             p->pkt_drop = DMA_PKT_DROP_DIS;
         }
     }
 
     return 0;
 }
 
 static void dma_free_desc_resource(struct virt_dma_desc *vdesc)
 {
     struct dw2_desc_sw *ds = to_lgm_dma_desc(vdesc);
     struct ldma_chan *c = ds->chan;
 
     dma_pool_free(c->desc_pool, ds->desc_hw, ds->desc_phys);
     kfree(ds);
 }
 
 static struct dw2_desc_sw *
 dma_alloc_desc_resource(int num, struct ldma_chan *c)
 {
     struct device *dev = c->vchan.chan.device->dev;
     struct dw2_desc_sw *ds;
 
     if (num > c->desc_num) {
         dev_err(dev, "sg num %d exceed max %d\n", num, c->desc_num);
         return NULL;
     }
 
     ds = kzalloc(sizeof(*ds), GFP_NOWAIT);
     if (!ds)
         return NULL;
 
     ds->chan = c;
     ds->desc_hw = dma_pool_zalloc(c->desc_pool, GFP_ATOMIC,
                       &ds->desc_phys);
     if (!ds->desc_hw) {
         dev_dbg(dev, "out of memory for link descriptor\n");
         kfree(ds);
         return NULL;
     }
     ds->desc_cnt = num;
 
     return ds;
 }
 
 static void ldma_chan_irq_en(struct ldma_chan *c)
 {
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     unsigned long flags;
 
     spin_lock_irqsave(&d->dev_lock, flags);
     writel(c->nr, d->base + DMA_CS);
     writel(DMA_CI_EOP, d->base + DMA_CIE);
     writel(BIT(c->nr), d->base + DMA_IRNEN);
     spin_unlock_irqrestore(&d->dev_lock, flags);
 }
 
 static void ldma_issue_pending(struct dma_chan *chan)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     unsigned long flags;
 
     if (d->ver == DMA_VER22) {
         spin_lock_irqsave(&c->vchan.lock, flags);
         if (vchan_issue_pending(&c->vchan)) {
             struct virt_dma_desc *vdesc;
 
             /* Get the next descriptor */
             vdesc = vchan_next_desc(&c->vchan);
             if (!vdesc) {
                 c->ds = NULL;
                 spin_unlock_irqrestore(&c->vchan.lock, flags);
                 return;
             }
             list_del(&vdesc->node);
             c->ds = to_lgm_dma_desc(vdesc);
             ldma_chan_desc_hw_cfg(c, c->ds->desc_phys, c->ds->desc_cnt);
             ldma_chan_irq_en(c);
         }
         spin_unlock_irqrestore(&c->vchan.lock, flags);
     }
     ldma_chan_on(c);
 }
 
 static void ldma_synchronize(struct dma_chan *chan)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
 
     /*
      * clear any pending work if any. In that
      * case the resource needs to be free here.
      */
     cancel_work_sync(&c->work);
     vchan_synchronize(&c->vchan);
     if (c->ds)
         dma_free_desc_resource(&c->ds->vdesc);
 }
 
 static int ldma_terminate_all(struct dma_chan *chan)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
     unsigned long flags;
     LIST_HEAD(head);
 
     spin_lock_irqsave(&c->vchan.lock, flags);
     vchan_get_all_descriptors(&c->vchan, &head);
     spin_unlock_irqrestore(&c->vchan.lock, flags);
     vchan_dma_desc_free_list(&c->vchan, &head);
 
     return ldma_chan_reset(c);
 }
 
 static int ldma_resume_chan(struct dma_chan *chan)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
 
     ldma_chan_on(c);
 
     return 0;
 }
 
 static int ldma_pause_chan(struct dma_chan *chan)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
 
     return ldma_chan_off(c);
 }
 
 static enum dma_status
 ldma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
            struct dma_tx_state *txstate)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     enum dma_status status = DMA_COMPLETE;
 
     if (d->ver == DMA_VER22)
         status = dma_cookie_status(chan, cookie, txstate);
 
     return status;
 }
 
 static void dma_chan_irq(int irq, void *data)
 {
     struct ldma_chan *c = data;
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     u32 stat;
 
     /* Disable channel interrupts  */
     writel(c->nr, d->base + DMA_CS);
     stat = readl(d->base + DMA_CIS);
     if (!stat)
         return;
 
     writel(readl(d->base + DMA_CIE) & ~DMA_CI_ALL, d->base + DMA_CIE);
     writel(stat, d->base + DMA_CIS);
     queue_work(d->wq, &c->work);
 }
 
 static irqreturn_t dma_interrupt(int irq, void *dev_id)
 {
     struct ldma_dev *d = dev_id;
     struct ldma_chan *c;
     unsigned long irncr;
     u32 cid;
 
     irncr = readl(d->base + DMA_IRNCR);
     if (!irncr) {
         dev_err(d->dev, "dummy interrupt\n");
         return IRQ_NONE;
     }
 
     for_each_set_bit(cid, &irncr, d->chan_nrs) {
         /* Mask */
         writel(readl(d->base + DMA_IRNEN) & ~BIT(cid), d->base + DMA_IRNEN);
         /* Ack */
         writel(readl(d->base + DMA_IRNCR) | BIT(cid), d->base + DMA_IRNCR);
 
         c = &d->chans[cid];
         dma_chan_irq(irq, c);
     }
 
     return IRQ_HANDLED;
 }
 
 static void prep_slave_burst_len(struct ldma_chan *c)
 {
     struct ldma_port *p = c->port;
     struct dma_slave_config *cfg = &c->config;
 
     if (cfg->dst_maxburst)
         cfg->src_maxburst = cfg->dst_maxburst;
 
     /* TX and RX has the same burst length */
     p->txbl = ilog2(cfg->src_maxburst);
     p->rxbl = p->txbl;
 }
 
 static struct dma_async_tx_descriptor *
 ldma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
            unsigned int sglen, enum dma_transfer_direction dir,
            unsigned long flags, void *context)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     size_t len, avail, total = 0;
     struct dw2_desc *hw_ds;
     struct dw2_desc_sw *ds;
     struct scatterlist *sg;
     int num = sglen, i;
     dma_addr_t addr;
 
     if (!sgl)
         return NULL;
 
     if (d->ver > DMA_VER22)
         return ldma_chan_desc_cfg(chan, sgl->dma_address, sglen);
 
     for_each_sg(sgl, sg, sglen, i) {
         avail = sg_dma_len(sg);
         if (avail > DMA_MAX_SIZE)
             num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1;
     }
 
     ds = dma_alloc_desc_resource(num, c);
     if (!ds)
         return NULL;
 
     c->ds = ds;
 
     num = 0;
     /* sop and eop has to be handled nicely */
     for_each_sg(sgl, sg, sglen, i) {
         addr = sg_dma_address(sg);
         avail = sg_dma_len(sg);
         total += avail;
 
         do {
             len = min_t(size_t, avail, DMA_MAX_SIZE);
 
             hw_ds = &ds->desc_hw[num];
             switch (sglen) {
             case 1:
                 hw_ds->field &= ~DESC_SOP;
                 hw_ds->field |= FIELD_PREP(DESC_SOP, 1);
 
                 hw_ds->field &= ~DESC_EOP;
                 hw_ds->field |= FIELD_PREP(DESC_EOP, 1);
                 break;
             default:
                 if (num == 0) {
                     hw_ds->field &= ~DESC_SOP;
                     hw_ds->field |= FIELD_PREP(DESC_SOP, 1);
 
                     hw_ds->field &= ~DESC_EOP;
                     hw_ds->field |= FIELD_PREP(DESC_EOP, 0);
                 } else if (num == (sglen - 1)) {
                     hw_ds->field &= ~DESC_SOP;
                     hw_ds->field |= FIELD_PREP(DESC_SOP, 0);
                     hw_ds->field &= ~DESC_EOP;
                     hw_ds->field |= FIELD_PREP(DESC_EOP, 1);
                 } else {
                     hw_ds->field &= ~DESC_SOP;
                     hw_ds->field |= FIELD_PREP(DESC_SOP, 0);
 
                     hw_ds->field &= ~DESC_EOP;
                     hw_ds->field |= FIELD_PREP(DESC_EOP, 0);
                 }
                 break;
             }
             /* Only 32 bit address supported */
             hw_ds->addr = (u32)addr;
 
             hw_ds->field &= ~DESC_DATA_LEN;
             hw_ds->field |= FIELD_PREP(DESC_DATA_LEN, len);
 
             hw_ds->field &= ~DESC_C;
             hw_ds->field |= FIELD_PREP(DESC_C, 0);
 
             hw_ds->field &= ~DESC_BYTE_OFF;
             hw_ds->field |= FIELD_PREP(DESC_BYTE_OFF, addr & 0x3);
 
             /* Ensure data ready before ownership change */
             wmb();
             hw_ds->field &= ~DESC_OWN;
             hw_ds->field |= FIELD_PREP(DESC_OWN, DMA_OWN);
 
             /* Ensure ownership changed before moving forward */
             wmb();
             num++;
             addr += len;
             avail -= len;
         } while (avail);
     }
 
     ds->size = total;
     prep_slave_burst_len(c);
 
     return vchan_tx_prep(&c->vchan, &ds->vdesc, DMA_CTRL_ACK);
 }
 
 static int
 ldma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
 
     memcpy(&c->config, cfg, sizeof(c->config));
 
     return 0;
 }
 
 static int ldma_alloc_chan_resources(struct dma_chan *chan)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
     struct device *dev = c->vchan.chan.device->dev;
     size_t  desc_sz;
 
     if (d->ver > DMA_VER22) {
         c->flags |= CHAN_IN_USE;
         return 0;
     }
 
     if (c->desc_pool)
         return c->desc_num;
 
     desc_sz = c->desc_num * sizeof(struct dw2_desc);
     c->desc_pool = dma_pool_create(c->name, dev, desc_sz,
                        __alignof__(struct dw2_desc), 0);
 
     if (!c->desc_pool) {
         dev_err(dev, "unable to allocate descriptor pool\n");
         return -ENOMEM;
     }
 
     return c->desc_num;
 }
 
 static void ldma_free_chan_resources(struct dma_chan *chan)
 {
     struct ldma_chan *c = to_ldma_chan(chan);
     struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device);
 
     if (d->ver == DMA_VER22) {
         dma_pool_destroy(c->desc_pool);
         c->desc_pool = NULL;
         vchan_free_chan_resources(to_virt_chan(chan));
         ldma_chan_reset(c);
     } else {
         c->flags &= ~CHAN_IN_USE;
     }
 }
 
 static void dma_work(struct work_struct *work)
 {
     struct ldma_chan *c = container_of(work, struct ldma_chan, work);
     struct dma_async_tx_descriptor *tx = &c->ds->vdesc.tx;
     struct virt_dma_chan *vc = &c->vchan;
     struct dmaengine_desc_callback cb;
     struct virt_dma_desc *vd, *_vd;
     unsigned long flags;
     LIST_HEAD(head);
 
     spin_lock_irqsave(&c->vchan.lock, flags);
     list_splice_tail_init(&vc->desc_completed, &head);
     spin_unlock_irqrestore(&c->vchan.lock, flags);
     dmaengine_desc_get_callback(tx, &cb);
     dma_cookie_complete(tx);
     dmaengine_desc_callback_invoke(&cb, NULL);
 
     list_for_each_entry_safe(vd, _vd, &head, node) {
         dmaengine_desc_get_callback(tx, &cb);
         dma_cookie_complete(tx);
         list_del(&vd->node);
         dmaengine_desc_callback_invoke(&cb, NULL);
 
         vchan_vdesc_fini(vd);
     }
     c->ds = NULL;
 }
 
 static void
 update_burst_len_v22(struct ldma_chan *c, struct ldma_port *p, u32 burst)
 {
     if (ldma_chan_tx(c))
         p->txbl = ilog2(burst);
     else
         p->rxbl = ilog2(burst);
 }
 
 static void
 update_burst_len_v3X(struct ldma_chan *c, struct ldma_port *p, u32 burst)
 {
     if (ldma_chan_tx(c))
         p->txbl = burst;
     else
         p->rxbl = burst;
 }
 
 static int
 update_client_configs(struct of_dma *ofdma, struct of_phandle_args *spec)
 {
     struct ldma_dev *d = ofdma->of_dma_data;
     u32 chan_id =  spec->args[0];
     u32 port_id =  spec->args[1];
     u32 burst = spec->args[2];
     struct ldma_port *p;
     struct ldma_chan *c;
 
     if (chan_id >= d->chan_nrs || port_id >= d->port_nrs)
         return 0;
 
     p = &d->ports[port_id];
     c = &d->chans[chan_id];
     c->port = p;
 
     if (d->ver == DMA_VER22)
         update_burst_len_v22(c, p, burst);
     else
         update_burst_len_v3X(c, p, burst);
 
     ldma_port_cfg(p);
 
     return 1;
 }
 
 static struct dma_chan *ldma_xlate(struct of_phandle_args *spec,
                    struct of_dma *ofdma)
 {
     struct ldma_dev *d = ofdma->of_dma_data;
     u32 chan_id =  spec->args[0];
     int ret;
 
     if (!spec->args_count)
         return NULL;
 
     /* if args_count is 1 driver use default settings */
     if (spec->args_count > 1) {
         ret = update_client_configs(ofdma, spec);
         if (!ret)
             return NULL;
     }
 
     return dma_get_slave_channel(&d->chans[chan_id].vchan.chan);
 }
 
 static void ldma_dma_init_v22(int i, struct ldma_dev *d)
 {
     struct ldma_chan *c;
 
     c = &d->chans[i];
     c->nr = i; /* Real channel number */
     c->rst = DMA_CHAN_RST;
     c->desc_num = DMA_DFT_DESC_NUM;
     snprintf(c->name, sizeof(c->name), "chan%d", c->nr);
     INIT_WORK(&c->work, dma_work);
     c->vchan.desc_free = dma_free_desc_resource;
     vchan_init(&c->vchan, &d->dma_dev);
 }
 
 static void ldma_dma_init_v3X(int i, struct ldma_dev *d)
 {
     struct ldma_chan *c;
 
     c = &d->chans[i];
     c->data_endian = DMA_DFT_ENDIAN;
     c->desc_endian = DMA_DFT_ENDIAN;
     c->data_endian_en = false;
     c->desc_endian_en = false;
     c->desc_rx_np = false;
     c->flags |= DEVICE_ALLOC_DESC;
     c->onoff = DMA_CH_OFF;
     c->rst = DMA_CHAN_RST;
     c->abc_en = true;
     c->hdrm_csum = false;
     c->boff_len = 0;
     c->nr = i;
     c->vchan.desc_free = dma_free_desc_resource;
     vchan_init(&c->vchan, &d->dma_dev);
 }
 
 static int ldma_init_v22(struct ldma_dev *d, struct platform_device *pdev)
 {
     int ret;
 
     ret = device_property_read_u32(d->dev, "dma-channels", &d->chan_nrs);
     if (ret < 0) {
         dev_err(d->dev, "unable to read dma-channels property\n");
         return ret;
     }
 
     d->irq = platform_get_irq(pdev, 0);
     if (d->irq < 0)
         return d->irq;
 
     ret = devm_request_irq(&pdev->dev, d->irq, dma_interrupt, 0,
                    DRIVER_NAME, d);
     if (ret)
         return ret;
 
     d->wq = alloc_ordered_workqueue("dma_wq", WQ_MEM_RECLAIM |
             WQ_HIGHPRI);
     if (!d->wq)
         return -ENOMEM;
 
     return 0;
 }
 
 static void ldma_clk_disable(void *data)
 {
     struct ldma_dev *d = data;
 
     clk_disable_unprepare(d->core_clk);
     reset_control_assert(d->rst);
 }
 
 static const struct ldma_inst_data dma0 = {
     .name = "dma0",
     .chan_fc = false,
     .desc_fod = false,
     .desc_in_sram = false,
     .valid_desc_fetch_ack = false,
 };
 
 static const struct ldma_inst_data dma2tx = {
     .name = "dma2tx",
     .type = DMA_TYPE_TX,
     .orrc = 16,
     .chan_fc = true,
     .desc_fod = true,
     .desc_in_sram = true,
     .valid_desc_fetch_ack = true,
 };
 
 static const struct ldma_inst_data dma1rx = {
     .name = "dma1rx",
     .type = DMA_TYPE_RX,
     .orrc = 16,
     .chan_fc = false,
     .desc_fod = true,
     .desc_in_sram = true,
     .valid_desc_fetch_ack = false,
 };
 
 static const struct ldma_inst_data dma1tx = {
     .name = "dma1tx",
     .type = DMA_TYPE_TX,
     .orrc = 16,
     .chan_fc = true,
     .desc_fod = true,
     .desc_in_sram = true,
     .valid_desc_fetch_ack = true,
 };
 
 static const struct ldma_inst_data dma0tx = {
     .name = "dma0tx",
     .type = DMA_TYPE_TX,
     .orrc = 16,
     .chan_fc = true,
     .desc_fod = true,
     .desc_in_sram = true,
     .valid_desc_fetch_ack = true,
 };
 
 static const struct ldma_inst_data dma3 = {
     .name = "dma3",
     .type = DMA_TYPE_MCPY,
     .orrc = 16,
     .chan_fc = false,
     .desc_fod = false,
     .desc_in_sram = true,
     .valid_desc_fetch_ack = false,
 };
 
 static const struct ldma_inst_data toe_dma30 = {
     .name = "toe_dma30",
     .type = DMA_TYPE_MCPY,
     .orrc = 16,
     .chan_fc = false,
     .desc_fod = false,
     .desc_in_sram = true,
     .valid_desc_fetch_ack = true,
 };
 
 static const struct ldma_inst_data toe_dma31 = {
     .name = "toe_dma31",
     .type = DMA_TYPE_MCPY,
     .orrc = 16,
     .chan_fc = false,
     .desc_fod = false,
     .desc_in_sram = true,
     .valid_desc_fetch_ack = true,
 };
 
 static const struct of_device_id intel_ldma_match[] = {
     { .compatible = "intel,lgm-cdma", .data = &dma0},
     { .compatible = "intel,lgm-dma2tx", .data = &dma2tx},
     { .compatible = "intel,lgm-dma1rx", .data = &dma1rx},
     { .compatible = "intel,lgm-dma1tx", .data = &dma1tx},
     { .compatible = "intel,lgm-dma0tx", .data = &dma0tx},
     { .compatible = "intel,lgm-dma3", .data = &dma3},
     { .compatible = "intel,lgm-toe-dma30", .data = &toe_dma30},
     { .compatible = "intel,lgm-toe-dma31", .data = &toe_dma31},
     {}
 };
 
 static int intel_ldma_probe(struct platform_device *pdev)
 {
     struct device *dev = &pdev->dev;
     struct dma_device *dma_dev;
     unsigned long ch_mask;
     struct ldma_chan *c;
     struct ldma_port *p;
     struct ldma_dev *d;
     u32 id, bitn = 32, j;
     int i, ret;
 
     d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);
     if (!d)
         return -ENOMEM;
 
     /* Link controller to platform device */
     d->dev = &pdev->dev;
 
     d->inst = device_get_match_data(dev);
     if (!d->inst) {
         dev_err(dev, "No device match found\n");
         return -ENODEV;
     }
 
     d->base = devm_platform_ioremap_resource(pdev, 0);
     if (IS_ERR(d->base))
         return PTR_ERR(d->base);
 
     /* Power up and reset the dma engine, some DMAs always on?? */
     d->core_clk = devm_clk_get_optional(dev, NULL);
     if (IS_ERR(d->core_clk))
         return PTR_ERR(d->core_clk);
 
     d->rst = devm_reset_control_get_optional(dev, NULL);
     if (IS_ERR(d->rst))
         return PTR_ERR(d->rst);
 
     clk_prepare_enable(d->core_clk);
     reset_control_deassert(d->rst);
 
     ret = devm_add_action_or_reset(dev, ldma_clk_disable, d);
     if (ret) {
         dev_err(dev, "Failed to devm_add_action_or_reset, %d\n", ret);
         return ret;
     }
 
     id = readl(d->base + DMA_ID);
     d->chan_nrs = FIELD_GET(DMA_ID_CHNR, id);
     d->port_nrs = FIELD_GET(DMA_ID_PNR, id);
     d->ver = FIELD_GET(DMA_ID_REV, id);
 
     if (id & DMA_ID_AW_36B)
         d->flags |= DMA_ADDR_36BIT;
 
     if (IS_ENABLED(CONFIG_64BIT) && (id & DMA_ID_AW_36B))
         bitn = 36;
 
     if (id & DMA_ID_DW_128B)
         d->flags |= DMA_DATA_128BIT;
 
     ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(bitn));
     if (ret) {
         dev_err(dev, "No usable DMA configuration\n");
         return ret;
     }
 
     if (d->ver == DMA_VER22) {
         ret = ldma_init_v22(d, pdev);
         if (ret)
             return ret;
     }
 
     ret = device_property_read_u32(dev, "dma-channel-mask", &d->channels_mask);
     if (ret < 0)
         d->channels_mask = GENMASK(d->chan_nrs - 1, 0);
 
     dma_dev = &d->dma_dev;
 
     dma_cap_zero(dma_dev->cap_mask);
     dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
 
     /* Channel initializations */
     INIT_LIST_HEAD(&dma_dev->channels);
 
     /* Port Initializations */
     d->ports = devm_kcalloc(dev, d->port_nrs, sizeof(*p), GFP_KERNEL);
     if (!d->ports)
         return -ENOMEM;
 
     /* Channels Initializations */
     d->chans = devm_kcalloc(d->dev, d->chan_nrs, sizeof(*c), GFP_KERNEL);
     if (!d->chans)
         return -ENOMEM;
 
     for (i = 0; i < d->port_nrs; i++) {
         p = &d->ports[i];
         p->portid = i;
         p->ldev = d;
     }
 
     ret = ldma_cfg_init(d);
     if (ret)
         return ret;
 
     dma_dev->dev = &pdev->dev;
 
     ch_mask = (unsigned long)d->channels_mask;
     for_each_set_bit(j, &ch_mask, d->chan_nrs) {
         if (d->ver == DMA_VER22)
             ldma_dma_init_v22(j, d);
         else
             ldma_dma_init_v3X(j, d);
     }
 
     dma_dev->device_alloc_chan_resources = ldma_alloc_chan_resources;
     dma_dev->device_free_chan_resources = ldma_free_chan_resources;
     dma_dev->device_terminate_all = ldma_terminate_all;
     dma_dev->device_issue_pending = ldma_issue_pending;
     dma_dev->device_tx_status = ldma_tx_status;
     dma_dev->device_resume = ldma_resume_chan;
     dma_dev->device_pause = ldma_pause_chan;
     dma_dev->device_prep_slave_sg = ldma_prep_slave_sg;
 
     if (d->ver == DMA_VER22) {
         dma_dev->device_config = ldma_slave_config;
         dma_dev->device_synchronize = ldma_synchronize;
         dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
         dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
         dma_dev->directions = BIT(DMA_MEM_TO_DEV) |
                       BIT(DMA_DEV_TO_MEM);
         dma_dev->residue_granularity =
                     DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
     }
 
     platform_set_drvdata(pdev, d);
 
     ldma_dev_init(d);
 
     ret = dma_async_device_register(dma_dev);
     if (ret) {
         dev_err(dev, "Failed to register slave DMA engine device\n");
         return ret;
     }
 
     ret = of_dma_controller_register(pdev->dev.of_node, ldma_xlate, d);
     if (ret) {
         dev_err(dev, "Failed to register of DMA controller\n");
         dma_async_device_unregister(dma_dev);
         return ret;
     }
 
     dev_info(dev, "Init done - rev: %x, ports: %d channels: %d\n", d->ver,
          d->port_nrs, d->chan_nrs);
 
     return 0;
 }
 
 static struct platform_driver intel_ldma_driver = {
     .probe = intel_ldma_probe,
     .driver = {
         .name = DRIVER_NAME,
         .of_match_table = intel_ldma_match,
     },
 };
 
 /*
  * Perform this driver as device_initcall to make sure initialization happens
  * before its DMA clients of some are platform specific and also to provide
  * registered DMA channels and DMA capabilities to clients before their
  * initialization.
  */
 static int __init intel_ldma_init(void)
 {
     return platform_driver_register(&intel_ldma_driver);
 }
 
 device_initcall(intel_ldma_init);

This page was automatically generated by the 2.1.0 LXR engine. The LXR team