Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * TI K3 NAVSS Ring Accelerator subsystem driver
0004  *
0005  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
0006  */
0007 
0008 #include <linux/dma-mapping.h>
0009 #include <linux/io.h>
0010 #include <linux/init.h>
0011 #include <linux/of.h>
0012 #include <linux/of_device.h>
0013 #include <linux/platform_device.h>
0014 #include <linux/sys_soc.h>
0015 #include <linux/dma/ti-cppi5.h>
0016 #include <linux/soc/ti/k3-ringacc.h>
0017 #include <linux/soc/ti/ti_sci_protocol.h>
0018 #include <linux/soc/ti/ti_sci_inta_msi.h>
0019 #include <linux/of_irq.h>
0020 #include <linux/irqdomain.h>
0021 
0022 static LIST_HEAD(k3_ringacc_list);
0023 static DEFINE_MUTEX(k3_ringacc_list_lock);
0024 
0025 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK     GENMASK(19, 0)
0026 #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK     GENMASK(15, 0)
0027 
0028 /**
0029  * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region
0030  *
0031  * @resv_16: Reserved
0032  * @db: Ring Doorbell Register
0033  * @resv_4: Reserved
0034  * @occ: Ring Occupancy Register
0035  * @indx: Ring Current Index Register
0036  * @hwocc: Ring Hardware Occupancy Register
0037  * @hwindx: Ring Hardware Current Index Register
0038  */
0039 struct k3_ring_rt_regs {
0040     u32 resv_16[4];
0041     u32 db;
0042     u32 resv_4[1];
0043     u32 occ;
0044     u32 indx;
0045     u32 hwocc;
0046     u32 hwindx;
0047 };
0048 
0049 #define K3_RINGACC_RT_REGS_STEP         0x1000
0050 #define K3_DMARING_RT_REGS_STEP         0x2000
0051 #define K3_DMARING_RT_REGS_REVERSE_OFS      0x1000
0052 #define K3_RINGACC_RT_OCC_MASK          GENMASK(20, 0)
0053 #define K3_DMARING_RT_OCC_TDOWN_COMPLETE    BIT(31)
0054 #define K3_DMARING_RT_DB_ENTRY_MASK     GENMASK(7, 0)
0055 #define K3_DMARING_RT_DB_TDOWN_ACK      BIT(31)
0056 
0057 /**
0058  * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region
0059  *
0060  * @head_data: Ring Head Entry Data Registers
0061  * @tail_data: Ring Tail Entry Data Registers
0062  * @peek_head_data: Ring Peek Head Entry Data Regs
0063  * @peek_tail_data: Ring Peek Tail Entry Data Regs
0064  */
0065 struct k3_ring_fifo_regs {
0066     u32 head_data[128];
0067     u32 tail_data[128];
0068     u32 peek_head_data[128];
0069     u32 peek_tail_data[128];
0070 };
0071 
0072 /**
0073  * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
0074  *
0075  * @revision: Revision Register
0076  * @config: Config Register
0077  */
0078 struct k3_ringacc_proxy_gcfg_regs {
0079     u32 revision;
0080     u32 config;
0081 };
0082 
0083 #define K3_RINGACC_PROXY_CFG_THREADS_MASK       GENMASK(15, 0)
0084 
0085 /**
0086  * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region
0087  *
0088  * @control: Proxy Control Register
0089  * @status: Proxy Status Register
0090  * @resv_512: Reserved
0091  * @data: Proxy Data Register
0092  */
0093 struct k3_ringacc_proxy_target_regs {
0094     u32 control;
0095     u32 status;
0096     u8  resv_512[504];
0097     u32 data[128];
0098 };
0099 
0100 #define K3_RINGACC_PROXY_TARGET_STEP    0x1000
0101 #define K3_RINGACC_PROXY_NOT_USED   (-1)
0102 
0103 enum k3_ringacc_proxy_access_mode {
0104     PROXY_ACCESS_MODE_HEAD = 0,
0105     PROXY_ACCESS_MODE_TAIL = 1,
0106     PROXY_ACCESS_MODE_PEEK_HEAD = 2,
0107     PROXY_ACCESS_MODE_PEEK_TAIL = 3,
0108 };
0109 
0110 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
0111 #define K3_RINGACC_FIFO_REGS_STEP   0x1000
0112 #define K3_RINGACC_MAX_DB_RING_CNT    (127U)
0113 
0114 struct k3_ring_ops {
0115     int (*push_tail)(struct k3_ring *ring, void *elm);
0116     int (*push_head)(struct k3_ring *ring, void *elm);
0117     int (*pop_tail)(struct k3_ring *ring, void *elm);
0118     int (*pop_head)(struct k3_ring *ring, void *elm);
0119 };
0120 
0121 /**
0122  * struct k3_ring_state - Internal state tracking structure
0123  *
0124  * @free: Number of free entries
0125  * @occ: Occupancy
0126  * @windex: Write index
0127  * @rindex: Read index
0128  */
0129 struct k3_ring_state {
0130     u32 free;
0131     u32 occ;
0132     u32 windex;
0133     u32 rindex;
0134     u32 tdown_complete:1;
0135 };
0136 
0137 /**
0138  * struct k3_ring - RA Ring descriptor
0139  *
0140  * @rt: Ring control/status registers
0141  * @fifos: Ring queues registers
0142  * @proxy: Ring Proxy Datapath registers
0143  * @ring_mem_dma: Ring buffer dma address
0144  * @ring_mem_virt: Ring buffer virt address
0145  * @ops: Ring operations
0146  * @size: Ring size in elements
0147  * @elm_size: Size of the ring element
0148  * @mode: Ring mode
0149  * @flags: flags
0150  * @state: Ring state
0151  * @ring_id: Ring Id
0152  * @parent: Pointer on struct @k3_ringacc
0153  * @use_count: Use count for shared rings
0154  * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
0155  * @dma_dev: device to be used for DMA API (allocation, mapping)
0156  * @asel: Address Space Select value for physical addresses
0157  */
0158 struct k3_ring {
0159     struct k3_ring_rt_regs __iomem *rt;
0160     struct k3_ring_fifo_regs __iomem *fifos;
0161     struct k3_ringacc_proxy_target_regs  __iomem *proxy;
0162     dma_addr_t  ring_mem_dma;
0163     void        *ring_mem_virt;
0164     struct k3_ring_ops *ops;
0165     u32     size;
0166     enum k3_ring_size elm_size;
0167     enum k3_ring_mode mode;
0168     u32     flags;
0169 #define K3_RING_FLAG_BUSY   BIT(1)
0170 #define K3_RING_FLAG_SHARED BIT(2)
0171 #define K3_RING_FLAG_REVERSE    BIT(3)
0172     struct k3_ring_state state;
0173     u32     ring_id;
0174     struct k3_ringacc   *parent;
0175     u32     use_count;
0176     int     proxy_id;
0177     struct device   *dma_dev;
0178     u32     asel;
0179 #define K3_ADDRESS_ASEL_SHIFT   48
0180 };
0181 
0182 struct k3_ringacc_ops {
0183     int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc);
0184 };
0185 
0186 /**
0187  * struct k3_ringacc - Rings accelerator descriptor
0188  *
0189  * @dev: pointer on RA device
0190  * @proxy_gcfg: RA proxy global config registers
0191  * @proxy_target_base: RA proxy datapath region
0192  * @num_rings: number of ring in RA
0193  * @rings_inuse: bitfield for ring usage tracking
0194  * @rm_gp_range: general purpose rings range from tisci
0195  * @dma_ring_reset_quirk: DMA reset w/a enable
0196  * @num_proxies: number of RA proxies
0197  * @proxy_inuse: bitfield for proxy usage tracking
0198  * @rings: array of rings descriptors (struct @k3_ring)
0199  * @list: list of RAs in the system
0200  * @req_lock: protect rings allocation
0201  * @tisci: pointer ti-sci handle
0202  * @tisci_ring_ops: ti-sci rings ops
0203  * @tisci_dev_id: ti-sci device id
0204  * @ops: SoC specific ringacc operation
0205  * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA)
0206  */
0207 struct k3_ringacc {
0208     struct device *dev;
0209     struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
0210     void __iomem *proxy_target_base;
0211     u32 num_rings; /* number of rings in Ringacc module */
0212     unsigned long *rings_inuse;
0213     struct ti_sci_resource *rm_gp_range;
0214 
0215     bool dma_ring_reset_quirk;
0216     u32 num_proxies;
0217     unsigned long *proxy_inuse;
0218 
0219     struct k3_ring *rings;
0220     struct list_head list;
0221     struct mutex req_lock; /* protect rings allocation */
0222 
0223     const struct ti_sci_handle *tisci;
0224     const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
0225     u32 tisci_dev_id;
0226 
0227     const struct k3_ringacc_ops *ops;
0228     bool dma_rings;
0229 };
0230 
0231 /**
0232  * struct k3_ringacc - Rings accelerator SoC data
0233  *
0234  * @dma_ring_reset_quirk:  DMA reset w/a enable
0235  */
0236 struct k3_ringacc_soc_data {
0237     unsigned dma_ring_reset_quirk:1;
0238 };
0239 
0240 static int k3_ringacc_ring_read_occ(struct k3_ring *ring)
0241 {
0242     return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK;
0243 }
0244 
0245 static void k3_ringacc_ring_update_occ(struct k3_ring *ring)
0246 {
0247     u32 val;
0248 
0249     val = readl(&ring->rt->occ);
0250 
0251     ring->state.occ = val & K3_RINGACC_RT_OCC_MASK;
0252     ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE);
0253 }
0254 
0255 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
0256 {
0257     return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
0258            (4 << ring->elm_size);
0259 }
0260 
0261 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
0262 {
0263     return (ring->ring_mem_virt + idx * (4 << ring->elm_size));
0264 }
0265 
0266 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
0267 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
0268 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem);
0269 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem);
0270 
0271 static struct k3_ring_ops k3_ring_mode_ring_ops = {
0272         .push_tail = k3_ringacc_ring_push_mem,
0273         .pop_head = k3_ringacc_ring_pop_mem,
0274 };
0275 
0276 static struct k3_ring_ops k3_dmaring_fwd_ops = {
0277         .push_tail = k3_ringacc_ring_push_mem,
0278         .pop_head = k3_dmaring_fwd_pop,
0279 };
0280 
0281 static struct k3_ring_ops k3_dmaring_reverse_ops = {
0282         /* Reverse side of the DMA ring can only be popped by SW */
0283         .pop_head = k3_dmaring_reverse_pop,
0284 };
0285 
0286 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
0287 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
0288 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
0289 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
0290 
0291 static struct k3_ring_ops k3_ring_mode_msg_ops = {
0292         .push_tail = k3_ringacc_ring_push_io,
0293         .push_head = k3_ringacc_ring_push_head_io,
0294         .pop_tail = k3_ringacc_ring_pop_tail_io,
0295         .pop_head = k3_ringacc_ring_pop_io,
0296 };
0297 
0298 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
0299 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
0300 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
0301 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
0302 
0303 static struct k3_ring_ops k3_ring_mode_proxy_ops = {
0304         .push_tail = k3_ringacc_ring_push_tail_proxy,
0305         .push_head = k3_ringacc_ring_push_head_proxy,
0306         .pop_tail = k3_ringacc_ring_pop_tail_proxy,
0307         .pop_head = k3_ringacc_ring_pop_head_proxy,
0308 };
0309 
0310 static void k3_ringacc_ring_dump(struct k3_ring *ring)
0311 {
0312     struct device *dev = ring->parent->dev;
0313 
0314     dev_dbg(dev, "dump ring: %d\n", ring->ring_id);
0315     dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt,
0316         &ring->ring_mem_dma);
0317     dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n",
0318         ring->elm_size, ring->size, ring->mode, ring->proxy_id);
0319     dev_dbg(dev, "dump flags %08X\n", ring->flags);
0320 
0321     dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db));
0322     dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ));
0323     dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx));
0324     dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc));
0325     dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx));
0326 
0327     if (ring->ring_mem_virt)
0328         print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE,
0329                      16, 1, ring->ring_mem_virt, 16 * 8, false);
0330 }
0331 
0332 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
0333                     int id, u32 flags)
0334 {
0335     int proxy_id = K3_RINGACC_PROXY_NOT_USED;
0336 
0337     mutex_lock(&ringacc->req_lock);
0338 
0339     if (id == K3_RINGACC_RING_ID_ANY) {
0340         /* Request for any general purpose ring */
0341         struct ti_sci_resource_desc *gp_rings =
0342                         &ringacc->rm_gp_range->desc[0];
0343         unsigned long size;
0344 
0345         size = gp_rings->start + gp_rings->num;
0346         id = find_next_zero_bit(ringacc->rings_inuse, size,
0347                     gp_rings->start);
0348         if (id == size)
0349             goto error;
0350     } else if (id < 0) {
0351         goto error;
0352     }
0353 
0354     if (test_bit(id, ringacc->rings_inuse) &&
0355         !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))
0356         goto error;
0357     else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)
0358         goto out;
0359 
0360     if (flags & K3_RINGACC_RING_USE_PROXY) {
0361         proxy_id = find_first_zero_bit(ringacc->proxy_inuse,
0362                           ringacc->num_proxies);
0363         if (proxy_id == ringacc->num_proxies)
0364             goto error;
0365     }
0366 
0367     if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
0368         set_bit(proxy_id, ringacc->proxy_inuse);
0369         ringacc->rings[id].proxy_id = proxy_id;
0370         dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,
0371             proxy_id);
0372     } else {
0373         dev_dbg(ringacc->dev, "Giving ring#%d\n", id);
0374     }
0375 
0376     set_bit(id, ringacc->rings_inuse);
0377 out:
0378     ringacc->rings[id].use_count++;
0379     mutex_unlock(&ringacc->req_lock);
0380     return &ringacc->rings[id];
0381 
0382 error:
0383     mutex_unlock(&ringacc->req_lock);
0384     return NULL;
0385 }
0386 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
0387 
0388 static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,
0389                     struct k3_ring **fwd_ring,
0390                     struct k3_ring **compl_ring)
0391 {
0392     int ret = 0;
0393 
0394     /*
0395      * DMA rings must be requested by ID, completion ring is the reverse
0396      * side of the forward ring
0397      */
0398     if (fwd_id < 0)
0399         return -EINVAL;
0400 
0401     mutex_lock(&ringacc->req_lock);
0402 
0403     if (test_bit(fwd_id, ringacc->rings_inuse)) {
0404         ret = -EBUSY;
0405         goto error;
0406     }
0407 
0408     *fwd_ring = &ringacc->rings[fwd_id];
0409     *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
0410     set_bit(fwd_id, ringacc->rings_inuse);
0411     ringacc->rings[fwd_id].use_count++;
0412     dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
0413 
0414     mutex_unlock(&ringacc->req_lock);
0415     return 0;
0416 
0417 error:
0418     mutex_unlock(&ringacc->req_lock);
0419     return ret;
0420 }
0421 
0422 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
0423                   int fwd_id, int compl_id,
0424                   struct k3_ring **fwd_ring,
0425                   struct k3_ring **compl_ring)
0426 {
0427     int ret = 0;
0428 
0429     if (!fwd_ring || !compl_ring)
0430         return -EINVAL;
0431 
0432     if (ringacc->dma_rings)
0433         return k3_dmaring_request_dual_ring(ringacc, fwd_id,
0434                             fwd_ring, compl_ring);
0435 
0436     *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);
0437     if (!(*fwd_ring))
0438         return -ENODEV;
0439 
0440     *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0);
0441     if (!(*compl_ring)) {
0442         k3_ringacc_ring_free(*fwd_ring);
0443         ret = -ENODEV;
0444     }
0445 
0446     return ret;
0447 }
0448 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair);
0449 
0450 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
0451 {
0452     struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
0453     struct k3_ringacc *ringacc = ring->parent;
0454     int ret;
0455 
0456     ring_cfg.nav_id = ringacc->tisci_dev_id;
0457     ring_cfg.index = ring->ring_id;
0458     ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID;
0459     ring_cfg.count = ring->size;
0460 
0461     ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
0462     if (ret)
0463         dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
0464             ret, ring->ring_id);
0465 }
0466 
0467 void k3_ringacc_ring_reset(struct k3_ring *ring)
0468 {
0469     if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
0470         return;
0471 
0472     memset(&ring->state, 0, sizeof(ring->state));
0473 
0474     k3_ringacc_ring_reset_sci(ring);
0475 }
0476 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset);
0477 
0478 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
0479                            enum k3_ring_mode mode)
0480 {
0481     struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
0482     struct k3_ringacc *ringacc = ring->parent;
0483     int ret;
0484 
0485     ring_cfg.nav_id = ringacc->tisci_dev_id;
0486     ring_cfg.index = ring->ring_id;
0487     ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID;
0488     ring_cfg.mode = mode;
0489 
0490     ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
0491     if (ret)
0492         dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
0493             ret, ring->ring_id);
0494 }
0495 
0496 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
0497 {
0498     if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
0499         return;
0500 
0501     if (!ring->parent->dma_ring_reset_quirk)
0502         goto reset;
0503 
0504     if (!occ)
0505         occ = k3_ringacc_ring_read_occ(ring);
0506 
0507     if (occ) {
0508         u32 db_ring_cnt, db_ring_cnt_cur;
0509 
0510         dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
0511             ring->ring_id, occ);
0512         /* TI-SCI ring reset */
0513         k3_ringacc_ring_reset_sci(ring);
0514 
0515         /*
0516          * Setup the ring in ring/doorbell mode (if not already in this
0517          * mode)
0518          */
0519         if (ring->mode != K3_RINGACC_RING_MODE_RING)
0520             k3_ringacc_ring_reconfig_qmode_sci(
0521                     ring, K3_RINGACC_RING_MODE_RING);
0522         /*
0523          * Ring the doorbell 2**22 – ringOcc times.
0524          * This will wrap the internal UDMAP ring state occupancy
0525          * counter (which is 21-bits wide) to 0.
0526          */
0527         db_ring_cnt = (1U << 22) - occ;
0528 
0529         while (db_ring_cnt != 0) {
0530             /*
0531              * Ring the doorbell with the maximum count each
0532              * iteration if possible to minimize the total
0533              * of writes
0534              */
0535             if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT)
0536                 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT;
0537             else
0538                 db_ring_cnt_cur = db_ring_cnt;
0539 
0540             writel(db_ring_cnt_cur, &ring->rt->db);
0541             db_ring_cnt -= db_ring_cnt_cur;
0542         }
0543 
0544         /* Restore the original ring mode (if not ring mode) */
0545         if (ring->mode != K3_RINGACC_RING_MODE_RING)
0546             k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
0547     }
0548 
0549 reset:
0550     /* Reset the ring */
0551     k3_ringacc_ring_reset(ring);
0552 }
0553 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma);
0554 
0555 static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
0556 {
0557     struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
0558     struct k3_ringacc *ringacc = ring->parent;
0559     int ret;
0560 
0561     ring_cfg.nav_id = ringacc->tisci_dev_id;
0562     ring_cfg.index = ring->ring_id;
0563     ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
0564 
0565     ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
0566     if (ret)
0567         dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
0568             ret, ring->ring_id);
0569 }
0570 
0571 int k3_ringacc_ring_free(struct k3_ring *ring)
0572 {
0573     struct k3_ringacc *ringacc;
0574 
0575     if (!ring)
0576         return -EINVAL;
0577 
0578     ringacc = ring->parent;
0579 
0580     /*
0581      * DMA rings: rings shared memory and configuration, only forward ring
0582      * is configured and reverse ring considered as slave.
0583      */
0584     if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
0585         return 0;
0586 
0587     dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
0588 
0589     if (!test_bit(ring->ring_id, ringacc->rings_inuse))
0590         return -EINVAL;
0591 
0592     mutex_lock(&ringacc->req_lock);
0593 
0594     if (--ring->use_count)
0595         goto out;
0596 
0597     if (!(ring->flags & K3_RING_FLAG_BUSY))
0598         goto no_init;
0599 
0600     k3_ringacc_ring_free_sci(ring);
0601 
0602     dma_free_coherent(ring->dma_dev,
0603               ring->size * (4 << ring->elm_size),
0604               ring->ring_mem_virt, ring->ring_mem_dma);
0605     ring->flags = 0;
0606     ring->ops = NULL;
0607     ring->dma_dev = NULL;
0608     ring->asel = 0;
0609 
0610     if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
0611         clear_bit(ring->proxy_id, ringacc->proxy_inuse);
0612         ring->proxy = NULL;
0613         ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
0614     }
0615 
0616 no_init:
0617     clear_bit(ring->ring_id, ringacc->rings_inuse);
0618 
0619 out:
0620     mutex_unlock(&ringacc->req_lock);
0621     return 0;
0622 }
0623 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free);
0624 
0625 u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
0626 {
0627     if (!ring)
0628         return -EINVAL;
0629 
0630     return ring->ring_id;
0631 }
0632 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id);
0633 
0634 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
0635 {
0636     if (!ring)
0637         return -EINVAL;
0638 
0639     return ring->parent->tisci_dev_id;
0640 }
0641 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id);
0642 
0643 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring)
0644 {
0645     int irq_num;
0646 
0647     if (!ring)
0648         return -EINVAL;
0649 
0650     irq_num = msi_get_virq(ring->parent->dev, ring->ring_id);
0651     if (irq_num <= 0)
0652         irq_num = -EINVAL;
0653     return irq_num;
0654 }
0655 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num);
0656 
0657 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
0658 {
0659     struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
0660     struct k3_ringacc *ringacc = ring->parent;
0661     int ret;
0662 
0663     if (!ringacc->tisci)
0664         return -EINVAL;
0665 
0666     ring_cfg.nav_id = ringacc->tisci_dev_id;
0667     ring_cfg.index = ring->ring_id;
0668     ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
0669     ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma);
0670     ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma);
0671     ring_cfg.count = ring->size;
0672     ring_cfg.mode = ring->mode;
0673     ring_cfg.size = ring->elm_size;
0674     ring_cfg.asel = ring->asel;
0675 
0676     ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
0677     if (ret)
0678         dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
0679             ret, ring->ring_id);
0680 
0681     return ret;
0682 }
0683 
0684 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
0685 {
0686     struct k3_ringacc *ringacc;
0687     struct k3_ring *reverse_ring;
0688     int ret = 0;
0689 
0690     if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 ||
0691         cfg->mode != K3_RINGACC_RING_MODE_RING ||
0692         cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK)
0693         return -EINVAL;
0694 
0695     ringacc = ring->parent;
0696 
0697     /*
0698      * DMA rings: rings shared memory and configuration, only forward ring
0699      * is configured and reverse ring considered as slave.
0700      */
0701     if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
0702         return 0;
0703 
0704     if (!test_bit(ring->ring_id, ringacc->rings_inuse))
0705         return -EINVAL;
0706 
0707     ring->size = cfg->size;
0708     ring->elm_size = cfg->elm_size;
0709     ring->mode = cfg->mode;
0710     ring->asel = cfg->asel;
0711     ring->dma_dev = cfg->dma_dev;
0712     if (!ring->dma_dev) {
0713         dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n",
0714              ring->ring_id);
0715         ring->dma_dev = ringacc->dev;
0716     }
0717 
0718     memset(&ring->state, 0, sizeof(ring->state));
0719 
0720     ring->ops = &k3_dmaring_fwd_ops;
0721 
0722     ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
0723                          ring->size * (4 << ring->elm_size),
0724                          &ring->ring_mem_dma, GFP_KERNEL);
0725     if (!ring->ring_mem_virt) {
0726         dev_err(ringacc->dev, "Failed to alloc ring mem\n");
0727         ret = -ENOMEM;
0728         goto err_free_ops;
0729     }
0730 
0731     ret = k3_ringacc_ring_cfg_sci(ring);
0732     if (ret)
0733         goto err_free_mem;
0734 
0735     ring->flags |= K3_RING_FLAG_BUSY;
0736 
0737     k3_ringacc_ring_dump(ring);
0738 
0739     /* DMA rings: configure reverse ring */
0740     reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
0741     reverse_ring->size = cfg->size;
0742     reverse_ring->elm_size = cfg->elm_size;
0743     reverse_ring->mode = cfg->mode;
0744     reverse_ring->asel = cfg->asel;
0745     memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
0746     reverse_ring->ops = &k3_dmaring_reverse_ops;
0747 
0748     reverse_ring->ring_mem_virt = ring->ring_mem_virt;
0749     reverse_ring->ring_mem_dma = ring->ring_mem_dma;
0750     reverse_ring->flags |= K3_RING_FLAG_BUSY;
0751     k3_ringacc_ring_dump(reverse_ring);
0752 
0753     return 0;
0754 
0755 err_free_mem:
0756     dma_free_coherent(ring->dma_dev,
0757               ring->size * (4 << ring->elm_size),
0758               ring->ring_mem_virt,
0759               ring->ring_mem_dma);
0760 err_free_ops:
0761     ring->ops = NULL;
0762     ring->proxy = NULL;
0763     ring->dma_dev = NULL;
0764     ring->asel = 0;
0765     return ret;
0766 }
0767 
0768 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
0769 {
0770     struct k3_ringacc *ringacc;
0771     int ret = 0;
0772 
0773     if (!ring || !cfg)
0774         return -EINVAL;
0775 
0776     ringacc = ring->parent;
0777 
0778     if (ringacc->dma_rings)
0779         return k3_dmaring_cfg(ring, cfg);
0780 
0781     if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
0782         cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
0783         cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
0784         !test_bit(ring->ring_id, ringacc->rings_inuse))
0785         return -EINVAL;
0786 
0787     if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE &&
0788         ring->proxy_id == K3_RINGACC_PROXY_NOT_USED &&
0789         cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) {
0790         dev_err(ringacc->dev,
0791             "Message mode must use proxy for %u element size\n",
0792             4 << ring->elm_size);
0793         return -EINVAL;
0794     }
0795 
0796     /*
0797      * In case of shared ring only the first user (master user) can
0798      * configure the ring. The sequence should be by the client:
0799      * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user
0800      * k3_ringacc_ring_cfg(ring, cfg); # master configuration
0801      * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
0802      * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
0803      */
0804     if (ring->use_count != 1)
0805         return 0;
0806 
0807     ring->size = cfg->size;
0808     ring->elm_size = cfg->elm_size;
0809     ring->mode = cfg->mode;
0810     memset(&ring->state, 0, sizeof(ring->state));
0811 
0812     if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
0813         ring->proxy = ringacc->proxy_target_base +
0814                   ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
0815 
0816     switch (ring->mode) {
0817     case K3_RINGACC_RING_MODE_RING:
0818         ring->ops = &k3_ring_mode_ring_ops;
0819         ring->dma_dev = cfg->dma_dev;
0820         if (!ring->dma_dev)
0821             ring->dma_dev = ringacc->dev;
0822         break;
0823     case K3_RINGACC_RING_MODE_MESSAGE:
0824         ring->dma_dev = ringacc->dev;
0825         if (ring->proxy)
0826             ring->ops = &k3_ring_mode_proxy_ops;
0827         else
0828             ring->ops = &k3_ring_mode_msg_ops;
0829         break;
0830     default:
0831         ring->ops = NULL;
0832         ret = -EINVAL;
0833         goto err_free_proxy;
0834     }
0835 
0836     ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
0837                          ring->size * (4 << ring->elm_size),
0838                          &ring->ring_mem_dma, GFP_KERNEL);
0839     if (!ring->ring_mem_virt) {
0840         dev_err(ringacc->dev, "Failed to alloc ring mem\n");
0841         ret = -ENOMEM;
0842         goto err_free_ops;
0843     }
0844 
0845     ret = k3_ringacc_ring_cfg_sci(ring);
0846 
0847     if (ret)
0848         goto err_free_mem;
0849 
0850     ring->flags |= K3_RING_FLAG_BUSY;
0851     ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
0852             K3_RING_FLAG_SHARED : 0;
0853 
0854     k3_ringacc_ring_dump(ring);
0855 
0856     return 0;
0857 
0858 err_free_mem:
0859     dma_free_coherent(ring->dma_dev,
0860               ring->size * (4 << ring->elm_size),
0861               ring->ring_mem_virt,
0862               ring->ring_mem_dma);
0863 err_free_ops:
0864     ring->ops = NULL;
0865     ring->dma_dev = NULL;
0866 err_free_proxy:
0867     ring->proxy = NULL;
0868     return ret;
0869 }
0870 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg);
0871 
0872 u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
0873 {
0874     if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
0875         return -EINVAL;
0876 
0877     return ring->size;
0878 }
0879 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size);
0880 
0881 u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
0882 {
0883     if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
0884         return -EINVAL;
0885 
0886     if (!ring->state.free)
0887         ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring);
0888 
0889     return ring->state.free;
0890 }
0891 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free);
0892 
0893 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
0894 {
0895     if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
0896         return -EINVAL;
0897 
0898     return k3_ringacc_ring_read_occ(ring);
0899 }
0900 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
0901 
0902 u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
0903 {
0904     return !k3_ringacc_ring_get_free(ring);
0905 }
0906 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full);
0907 
0908 enum k3_ringacc_access_mode {
0909     K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
0910     K3_RINGACC_ACCESS_MODE_POP_HEAD,
0911     K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
0912     K3_RINGACC_ACCESS_MODE_POP_TAIL,
0913     K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
0914     K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
0915 };
0916 
0917 #define K3_RINGACC_PROXY_MODE(x)    (((x) & 0x3) << 16)
0918 #define K3_RINGACC_PROXY_ELSIZE(x)  (((x) & 0x7) << 24)
0919 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
0920                      enum k3_ringacc_proxy_access_mode mode)
0921 {
0922     u32 val;
0923 
0924     val = ring->ring_id;
0925     val |= K3_RINGACC_PROXY_MODE(mode);
0926     val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size);
0927     writel(val, &ring->proxy->control);
0928     return 0;
0929 }
0930 
0931 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
0932                     enum k3_ringacc_access_mode access_mode)
0933 {
0934     void __iomem *ptr;
0935 
0936     ptr = (void __iomem *)&ring->proxy->data;
0937 
0938     switch (access_mode) {
0939     case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
0940     case K3_RINGACC_ACCESS_MODE_POP_HEAD:
0941         k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
0942         break;
0943     case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
0944     case K3_RINGACC_ACCESS_MODE_POP_TAIL:
0945         k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
0946         break;
0947     default:
0948         return -EINVAL;
0949     }
0950 
0951     ptr += k3_ringacc_ring_get_fifo_pos(ring);
0952 
0953     switch (access_mode) {
0954     case K3_RINGACC_ACCESS_MODE_POP_HEAD:
0955     case K3_RINGACC_ACCESS_MODE_POP_TAIL:
0956         dev_dbg(ring->parent->dev,
0957             "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
0958             access_mode);
0959         memcpy_fromio(elem, ptr, (4 << ring->elm_size));
0960         ring->state.occ--;
0961         break;
0962     case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
0963     case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
0964         dev_dbg(ring->parent->dev,
0965             "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
0966             access_mode);
0967         memcpy_toio(ptr, elem, (4 << ring->elm_size));
0968         ring->state.free--;
0969         break;
0970     default:
0971         return -EINVAL;
0972     }
0973 
0974     dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free,
0975         ring->state.occ);
0976     return 0;
0977 }
0978 
0979 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
0980 {
0981     return k3_ringacc_ring_access_proxy(ring, elem,
0982                         K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
0983 }
0984 
0985 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
0986 {
0987     return k3_ringacc_ring_access_proxy(ring, elem,
0988                         K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
0989 }
0990 
0991 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
0992 {
0993     return k3_ringacc_ring_access_proxy(ring, elem,
0994                         K3_RINGACC_ACCESS_MODE_POP_HEAD);
0995 }
0996 
0997 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
0998 {
0999     return k3_ringacc_ring_access_proxy(ring, elem,
1000                         K3_RINGACC_ACCESS_MODE_POP_HEAD);
1001 }
1002 
1003 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
1004                      enum k3_ringacc_access_mode access_mode)
1005 {
1006     void __iomem *ptr;
1007 
1008     switch (access_mode) {
1009     case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1010     case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1011         ptr = (void __iomem *)&ring->fifos->head_data;
1012         break;
1013     case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1014     case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1015         ptr = (void __iomem *)&ring->fifos->tail_data;
1016         break;
1017     default:
1018         return -EINVAL;
1019     }
1020 
1021     ptr += k3_ringacc_ring_get_fifo_pos(ring);
1022 
1023     switch (access_mode) {
1024     case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1025     case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1026         dev_dbg(ring->parent->dev,
1027             "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
1028             access_mode);
1029         memcpy_fromio(elem, ptr, (4 << ring->elm_size));
1030         ring->state.occ--;
1031         break;
1032     case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1033     case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1034         dev_dbg(ring->parent->dev,
1035             "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
1036             access_mode);
1037         memcpy_toio(ptr, elem, (4 << ring->elm_size));
1038         ring->state.free--;
1039         break;
1040     default:
1041         return -EINVAL;
1042     }
1043 
1044     dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n",
1045         ring->state.free, ring->state.windex, ring->state.occ,
1046         ring->state.rindex);
1047     return 0;
1048 }
1049 
1050 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
1051 {
1052     return k3_ringacc_ring_access_io(ring, elem,
1053                      K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
1054 }
1055 
1056 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
1057 {
1058     return k3_ringacc_ring_access_io(ring, elem,
1059                      K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
1060 }
1061 
1062 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
1063 {
1064     return k3_ringacc_ring_access_io(ring, elem,
1065                      K3_RINGACC_ACCESS_MODE_POP_HEAD);
1066 }
1067 
1068 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
1069 {
1070     return k3_ringacc_ring_access_io(ring, elem,
1071                      K3_RINGACC_ACCESS_MODE_POP_HEAD);
1072 }
1073 
1074 /*
1075  * The element is 48 bits of address + ASEL bits in the ring.
1076  * ASEL is used by the DMAs and should be removed for the kernel as it is not
1077  * part of the physical memory address.
1078  */
1079 static void k3_dmaring_remove_asel_from_elem(u64 *elem)
1080 {
1081     *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1082 }
1083 
1084 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem)
1085 {
1086     void *elem_ptr;
1087     u32 elem_idx;
1088 
1089     /*
1090      * DMA rings: forward ring is always tied DMA channel and HW does not
1091      * maintain any state data required for POP operation and its unknown
1092      * how much elements were consumed by HW. So, to actually
1093      * do POP, the read pointer has to be recalculated every time.
1094      */
1095     ring->state.occ = k3_ringacc_ring_read_occ(ring);
1096     if (ring->state.windex >= ring->state.occ)
1097         elem_idx = ring->state.windex - ring->state.occ;
1098     else
1099         elem_idx = ring->size - (ring->state.occ - ring->state.windex);
1100 
1101     elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx);
1102     memcpy(elem, elem_ptr, (4 << ring->elm_size));
1103     k3_dmaring_remove_asel_from_elem(elem);
1104 
1105     ring->state.occ--;
1106     writel(-1, &ring->rt->db);
1107 
1108     dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
1109         __func__, ring->state.occ, ring->state.windex, elem_idx,
1110         elem_ptr);
1111     return 0;
1112 }
1113 
1114 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem)
1115 {
1116     void *elem_ptr;
1117 
1118     elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1119 
1120     if (ring->state.occ) {
1121         memcpy(elem, elem_ptr, (4 << ring->elm_size));
1122         k3_dmaring_remove_asel_from_elem(elem);
1123 
1124         ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1125         ring->state.occ--;
1126         writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db);
1127     } else if (ring->state.tdown_complete) {
1128         dma_addr_t *value = elem;
1129 
1130         *value = CPPI5_TDCM_MARKER;
1131         writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db);
1132         ring->state.tdown_complete = false;
1133     }
1134 
1135     dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
1136         __func__, ring->state.occ, ring->state.rindex, elem_ptr);
1137     return 0;
1138 }
1139 
1140 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
1141 {
1142     void *elem_ptr;
1143 
1144     elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
1145 
1146     memcpy(elem_ptr, elem, (4 << ring->elm_size));
1147     if (ring->parent->dma_rings) {
1148         u64 *addr = elem_ptr;
1149 
1150         *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT);
1151     }
1152 
1153     ring->state.windex = (ring->state.windex + 1) % ring->size;
1154     ring->state.free--;
1155     writel(1, &ring->rt->db);
1156 
1157     dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
1158         ring->state.free, ring->state.windex);
1159 
1160     return 0;
1161 }
1162 
1163 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
1164 {
1165     void *elem_ptr;
1166 
1167     elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1168 
1169     memcpy(elem, elem_ptr, (4 << ring->elm_size));
1170 
1171     ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1172     ring->state.occ--;
1173     writel(-1, &ring->rt->db);
1174 
1175     dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
1176         ring->state.occ, ring->state.rindex, elem_ptr);
1177     return 0;
1178 }
1179 
1180 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
1181 {
1182     int ret = -EOPNOTSUPP;
1183 
1184     if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1185         return -EINVAL;
1186 
1187     dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n",
1188         ring->state.free, ring->state.windex);
1189 
1190     if (k3_ringacc_ring_is_full(ring))
1191         return -ENOMEM;
1192 
1193     if (ring->ops && ring->ops->push_tail)
1194         ret = ring->ops->push_tail(ring, elem);
1195 
1196     return ret;
1197 }
1198 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push);
1199 
1200 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
1201 {
1202     int ret = -EOPNOTSUPP;
1203 
1204     if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1205         return -EINVAL;
1206 
1207     dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
1208         ring->state.free, ring->state.windex);
1209 
1210     if (k3_ringacc_ring_is_full(ring))
1211         return -ENOMEM;
1212 
1213     if (ring->ops && ring->ops->push_head)
1214         ret = ring->ops->push_head(ring, elem);
1215 
1216     return ret;
1217 }
1218 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head);
1219 
1220 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
1221 {
1222     int ret = -EOPNOTSUPP;
1223 
1224     if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1225         return -EINVAL;
1226 
1227     if (!ring->state.occ)
1228         k3_ringacc_ring_update_occ(ring);
1229 
1230     dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
1231         ring->state.rindex);
1232 
1233     if (!ring->state.occ && !ring->state.tdown_complete)
1234         return -ENODATA;
1235 
1236     if (ring->ops && ring->ops->pop_head)
1237         ret = ring->ops->pop_head(ring, elem);
1238 
1239     return ret;
1240 }
1241 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop);
1242 
1243 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
1244 {
1245     int ret = -EOPNOTSUPP;
1246 
1247     if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1248         return -EINVAL;
1249 
1250     if (!ring->state.occ)
1251         k3_ringacc_ring_update_occ(ring);
1252 
1253     dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
1254         ring->state.occ, ring->state.rindex);
1255 
1256     if (!ring->state.occ)
1257         return -ENODATA;
1258 
1259     if (ring->ops && ring->ops->pop_tail)
1260         ret = ring->ops->pop_tail(ring, elem);
1261 
1262     return ret;
1263 }
1264 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail);
1265 
1266 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
1267                         const char *property)
1268 {
1269     struct device_node *ringacc_np;
1270     struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);
1271     struct k3_ringacc *entry;
1272 
1273     ringacc_np = of_parse_phandle(np, property, 0);
1274     if (!ringacc_np)
1275         return ERR_PTR(-ENODEV);
1276 
1277     mutex_lock(&k3_ringacc_list_lock);
1278     list_for_each_entry(entry, &k3_ringacc_list, list)
1279         if (entry->dev->of_node == ringacc_np) {
1280             ringacc = entry;
1281             break;
1282         }
1283     mutex_unlock(&k3_ringacc_list_lock);
1284     of_node_put(ringacc_np);
1285 
1286     return ringacc;
1287 }
1288 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle);
1289 
1290 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)
1291 {
1292     struct device_node *node = ringacc->dev->of_node;
1293     struct device *dev = ringacc->dev;
1294     struct platform_device *pdev = to_platform_device(dev);
1295     int ret;
1296 
1297     if (!node) {
1298         dev_err(dev, "device tree info unavailable\n");
1299         return -ENODEV;
1300     }
1301 
1302     ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);
1303     if (ret) {
1304         dev_err(dev, "ti,num-rings read failure %d\n", ret);
1305         return ret;
1306     }
1307 
1308     ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");
1309     if (IS_ERR(ringacc->tisci)) {
1310         ret = PTR_ERR(ringacc->tisci);
1311         if (ret != -EPROBE_DEFER)
1312             dev_err(dev, "ti,sci read fail %d\n", ret);
1313         ringacc->tisci = NULL;
1314         return ret;
1315     }
1316 
1317     ret = of_property_read_u32(node, "ti,sci-dev-id",
1318                    &ringacc->tisci_dev_id);
1319     if (ret) {
1320         dev_err(dev, "ti,sci-dev-id read fail %d\n", ret);
1321         return ret;
1322     }
1323 
1324     pdev->id = ringacc->tisci_dev_id;
1325 
1326     ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,
1327                         ringacc->tisci_dev_id,
1328                         "ti,sci-rm-range-gp-rings");
1329     if (IS_ERR(ringacc->rm_gp_range)) {
1330         dev_err(dev, "Failed to allocate MSI interrupts\n");
1331         return PTR_ERR(ringacc->rm_gp_range);
1332     }
1333 
1334     return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev,
1335                          ringacc->rm_gp_range);
1336 }
1337 
1338 static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = {
1339     .dma_ring_reset_quirk = 1,
1340 };
1341 
1342 static const struct soc_device_attribute k3_ringacc_socinfo[] = {
1343     { .family = "AM65X",
1344       .revision = "SR1.0",
1345       .data = &k3_ringacc_soc_data_sr1
1346     },
1347     {/* sentinel */}
1348 };
1349 
1350 static int k3_ringacc_init(struct platform_device *pdev,
1351                struct k3_ringacc *ringacc)
1352 {
1353     const struct soc_device_attribute *soc;
1354     void __iomem *base_fifo, *base_rt;
1355     struct device *dev = &pdev->dev;
1356     struct resource *res;
1357     int ret, i;
1358 
1359     dev->msi.domain = of_msi_get_domain(dev, dev->of_node,
1360                         DOMAIN_BUS_TI_SCI_INTA_MSI);
1361     if (!dev->msi.domain) {
1362         dev_err(dev, "Failed to get MSI domain\n");
1363         return -EPROBE_DEFER;
1364     }
1365 
1366     ret = k3_ringacc_probe_dt(ringacc);
1367     if (ret)
1368         return ret;
1369 
1370     soc = soc_device_match(k3_ringacc_socinfo);
1371     if (soc && soc->data) {
1372         const struct k3_ringacc_soc_data *soc_data = soc->data;
1373 
1374         ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk;
1375     }
1376 
1377     res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt");
1378     base_rt = devm_ioremap_resource(dev, res);
1379     if (IS_ERR(base_rt))
1380         return PTR_ERR(base_rt);
1381 
1382     res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos");
1383     base_fifo = devm_ioremap_resource(dev, res);
1384     if (IS_ERR(base_fifo))
1385         return PTR_ERR(base_fifo);
1386 
1387     res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg");
1388     ringacc->proxy_gcfg = devm_ioremap_resource(dev, res);
1389     if (IS_ERR(ringacc->proxy_gcfg))
1390         return PTR_ERR(ringacc->proxy_gcfg);
1391 
1392     res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1393                        "proxy_target");
1394     ringacc->proxy_target_base = devm_ioremap_resource(dev, res);
1395     if (IS_ERR(ringacc->proxy_target_base))
1396         return PTR_ERR(ringacc->proxy_target_base);
1397 
1398     ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) &
1399                      K3_RINGACC_PROXY_CFG_THREADS_MASK;
1400 
1401     ringacc->rings = devm_kzalloc(dev,
1402                       sizeof(*ringacc->rings) *
1403                       ringacc->num_rings,
1404                       GFP_KERNEL);
1405     ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings,
1406                           GFP_KERNEL);
1407     ringacc->proxy_inuse = devm_bitmap_zalloc(dev, ringacc->num_proxies,
1408                           GFP_KERNEL);
1409 
1410     if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1411         return -ENOMEM;
1412 
1413     for (i = 0; i < ringacc->num_rings; i++) {
1414         ringacc->rings[i].rt = base_rt +
1415                        K3_RINGACC_RT_REGS_STEP * i;
1416         ringacc->rings[i].fifos = base_fifo +
1417                       K3_RINGACC_FIFO_REGS_STEP * i;
1418         ringacc->rings[i].parent = ringacc;
1419         ringacc->rings[i].ring_id = i;
1420         ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1421     }
1422 
1423     ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1424 
1425     dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1426          ringacc->num_rings,
1427          ringacc->rm_gp_range->desc[0].start,
1428          ringacc->rm_gp_range->desc[0].num,
1429          ringacc->tisci_dev_id);
1430     dev_info(dev, "dma-ring-reset-quirk: %s\n",
1431          ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1432     dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1433          readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies);
1434 
1435     return 0;
1436 }
1437 
1438 struct ringacc_match_data {
1439     struct k3_ringacc_ops ops;
1440 };
1441 
1442 static struct ringacc_match_data k3_ringacc_data = {
1443     .ops = {
1444         .init = k3_ringacc_init,
1445     },
1446 };
1447 
1448 /* Match table for of_platform binding */
1449 static const struct of_device_id k3_ringacc_of_match[] = {
1450     { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, },
1451     {},
1452 };
1453 
1454 struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev,
1455                         struct k3_ringacc_init_data *data)
1456 {
1457     struct device *dev = &pdev->dev;
1458     struct k3_ringacc *ringacc;
1459     void __iomem *base_rt;
1460     struct resource *res;
1461     int i;
1462 
1463     ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1464     if (!ringacc)
1465         return ERR_PTR(-ENOMEM);
1466 
1467     ringacc->dev = dev;
1468     ringacc->dma_rings = true;
1469     ringacc->num_rings = data->num_rings;
1470     ringacc->tisci = data->tisci;
1471     ringacc->tisci_dev_id = data->tisci_dev_id;
1472 
1473     mutex_init(&ringacc->req_lock);
1474 
1475     res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt");
1476     base_rt = devm_ioremap_resource(dev, res);
1477     if (IS_ERR(base_rt))
1478         return ERR_CAST(base_rt);
1479 
1480     ringacc->rings = devm_kzalloc(dev,
1481                       sizeof(*ringacc->rings) *
1482                       ringacc->num_rings * 2,
1483                       GFP_KERNEL);
1484     ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings,
1485                           GFP_KERNEL);
1486 
1487     if (!ringacc->rings || !ringacc->rings_inuse)
1488         return ERR_PTR(-ENOMEM);
1489 
1490     for (i = 0; i < ringacc->num_rings; i++) {
1491         struct k3_ring *ring = &ringacc->rings[i];
1492 
1493         ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i;
1494         ring->parent = ringacc;
1495         ring->ring_id = i;
1496         ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1497 
1498         ring = &ringacc->rings[ringacc->num_rings + i];
1499         ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i +
1500                K3_DMARING_RT_REGS_REVERSE_OFS;
1501         ring->parent = ringacc;
1502         ring->ring_id = i;
1503         ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1504         ring->flags = K3_RING_FLAG_REVERSE;
1505     }
1506 
1507     ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1508 
1509     dev_info(dev, "Number of rings: %u\n", ringacc->num_rings);
1510 
1511     return ringacc;
1512 }
1513 EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init);
1514 
1515 static int k3_ringacc_probe(struct platform_device *pdev)
1516 {
1517     const struct ringacc_match_data *match_data;
1518     struct device *dev = &pdev->dev;
1519     struct k3_ringacc *ringacc;
1520     int ret;
1521 
1522     match_data = of_device_get_match_data(&pdev->dev);
1523     if (!match_data)
1524         return -ENODEV;
1525 
1526     ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1527     if (!ringacc)
1528         return -ENOMEM;
1529 
1530     ringacc->dev = dev;
1531     mutex_init(&ringacc->req_lock);
1532     ringacc->ops = &match_data->ops;
1533 
1534     ret = ringacc->ops->init(pdev, ringacc);
1535     if (ret)
1536         return ret;
1537 
1538     dev_set_drvdata(dev, ringacc);
1539 
1540     mutex_lock(&k3_ringacc_list_lock);
1541     list_add_tail(&ringacc->list, &k3_ringacc_list);
1542     mutex_unlock(&k3_ringacc_list_lock);
1543 
1544     return 0;
1545 }
1546 
1547 static struct platform_driver k3_ringacc_driver = {
1548     .probe      = k3_ringacc_probe,
1549     .driver     = {
1550         .name   = "k3-ringacc",
1551         .of_match_table = k3_ringacc_of_match,
1552         .suppress_bind_attrs = true,
1553     },
1554 };
1555 builtin_platform_driver(k3_ringacc_driver);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!