dma/ioat/dma.h

0001 /* SPDX-License-Identifier: GPL-2.0-or-later */
0002 /*
0003  * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
0004  */
0005 #ifndef IOATDMA_H
0006 #define IOATDMA_H
0007
0008 #include <linux/dmaengine.h>
0009 #include <linux/init.h>
0010 #include <linux/dmapool.h>
0011 #include <linux/cache.h>
0012 #include <linux/pci_ids.h>
0013 #include <linux/circ_buf.h>
0014 #include <linux/interrupt.h>
0015 #include "registers.h"
0016 #include "hw.h"
0017
0018 #define IOAT_DMA_VERSION  "5.00"
0019
0020 #define IOAT_DMA_DCA_ANY_CPU        ~0
0021
0022 #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, dma_dev)
0023 #define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev)
0024 #define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev)
0025
0026 #define chan_num(ch) ((int)((ch)->reg_base - (ch)->ioat_dma->reg_base) / 0x80)
0027
0028 /* ioat hardware assumes at least two sources for raid operations */
0029 #define src_cnt_to_sw(x) ((x) + 2)
0030 #define src_cnt_to_hw(x) ((x) - 2)
0031 #define ndest_to_sw(x) ((x) + 1)
0032 #define ndest_to_hw(x) ((x) - 1)
0033 #define src16_cnt_to_sw(x) ((x) + 9)
0034 #define src16_cnt_to_hw(x) ((x) - 9)
0035
0036 /*
0037  * workaround for IOAT ver.3.0 null descriptor issue
0038  * (channel returns error when size is 0)
0039  */
0040 #define NULL_DESC_BUFFER_SIZE 1
0041
0042 enum ioat_irq_mode {
0043     IOAT_NOIRQ = 0,
0044     IOAT_MSIX,
0045     IOAT_MSI,
0046     IOAT_INTX
0047 };
0048
0049 /**
0050  * struct ioatdma_device - internal representation of a IOAT device
0051  * @pdev: PCI-Express device
0052  * @reg_base: MMIO register space base address
0053  * @completion_pool: DMA buffers for completion ops
0054  * @sed_hw_pool: DMA super descriptor pools
0055  * @dma_dev: embedded struct dma_device
0056  * @version: version of ioatdma device
0057  * @msix_entries: irq handlers
0058  * @idx: per channel data
0059  * @dca: direct cache access context
0060  * @irq_mode: interrupt mode (INTX, MSI, MSIX)
0061  * @cap: read DMA capabilities register
0062  */
0063 struct ioatdma_device {
0064     struct pci_dev *pdev;
0065     void __iomem *reg_base;
0066     struct dma_pool *completion_pool;
0067 #define MAX_SED_POOLS   5
0068     struct dma_pool *sed_hw_pool[MAX_SED_POOLS];
0069     struct dma_device dma_dev;
0070     u8 version;
0071 #define IOAT_MAX_CHANS 4
0072     struct msix_entry msix_entries[IOAT_MAX_CHANS];
0073     struct ioatdma_chan *idx[IOAT_MAX_CHANS];
0074     struct dca_provider *dca;
0075     enum ioat_irq_mode irq_mode;
0076     u32 cap;
0077
0078     /* shadow version for CB3.3 chan reset errata workaround */
0079     u64 msixtba0;
0080     u64 msixdata0;
0081     u32 msixpba;
0082 };
0083
0084 #define IOAT_MAX_ORDER 16
0085 #define IOAT_MAX_DESCS (1 << IOAT_MAX_ORDER)
0086 #define IOAT_CHUNK_SIZE (SZ_512K)
0087 #define IOAT_DESCS_PER_CHUNK (IOAT_CHUNK_SIZE / IOAT_DESC_SZ)
0088
0089 struct ioat_descs {
0090     void *virt;
0091     dma_addr_t hw;
0092 };
0093
0094 struct ioatdma_chan {
0095     struct dma_chan dma_chan;
0096     void __iomem *reg_base;
0097     dma_addr_t last_completion;
0098     spinlock_t cleanup_lock;
0099     unsigned long state;
0100     #define IOAT_CHAN_DOWN 0
0101     #define IOAT_COMPLETION_ACK 1
0102     #define IOAT_RESET_PENDING 2
0103     #define IOAT_KOBJ_INIT_FAIL 3
0104     #define IOAT_RUN 5
0105     #define IOAT_CHAN_ACTIVE 6
0106     struct timer_list timer;
0107     #define RESET_DELAY msecs_to_jiffies(100)
0108     struct ioatdma_device *ioat_dma;
0109     dma_addr_t completion_dma;
0110     u64 *completion;
0111     struct tasklet_struct cleanup_task;
0112     struct kobject kobj;
0113
0114 /* ioat v2 / v3 channel attributes
0115  * @xfercap_log; log2 of channel max transfer length (for fast division)
0116  * @head: allocated index
0117  * @issued: hardware notification point
0118  * @tail: cleanup index
0119  * @dmacount: identical to 'head' except for occasionally resetting to zero
0120  * @alloc_order: log2 of the number of allocated descriptors
0121  * @produce: number of descriptors to produce at submit time
0122  * @ring: software ring buffer implementation of hardware ring
0123  * @prep_lock: serializes descriptor preparation (producers)
0124  */
0125     size_t xfercap_log;
0126     u16 head;
0127     u16 issued;
0128     u16 tail;
0129     u16 dmacount;
0130     u16 alloc_order;
0131     u16 produce;
0132     struct ioat_ring_ent **ring;
0133     spinlock_t prep_lock;
0134     struct ioat_descs descs[IOAT_MAX_DESCS / IOAT_DESCS_PER_CHUNK];
0135     int desc_chunks;
0136     int intr_coalesce;
0137     int prev_intr_coalesce;
0138 };
0139
0140 struct ioat_sysfs_entry {
0141     struct attribute attr;
0142     ssize_t (*show)(struct dma_chan *, char *);
0143     ssize_t (*store)(struct dma_chan *, const char *, size_t);
0144 };
0145
0146 /**
0147  * struct ioat_sed_ent - wrapper around super extended hardware descriptor
0148  * @hw: hardware SED
0149  * @dma: dma address for the SED
0150  * @parent: point to the dma descriptor that's the parent
0151  * @hw_pool: descriptor pool index
0152  */
0153 struct ioat_sed_ent {
0154     struct ioat_sed_raw_descriptor *hw;
0155     dma_addr_t dma;
0156     struct ioat_ring_ent *parent;
0157     unsigned int hw_pool;
0158 };
0159
0160 /**
0161  * struct ioat_ring_ent - wrapper around hardware descriptor
0162  * @hw: hardware DMA descriptor (for memcpy)
0163  * @xor: hardware xor descriptor
0164  * @xor_ex: hardware xor extension descriptor
0165  * @pq: hardware pq descriptor
0166  * @pq_ex: hardware pq extension descriptor
0167  * @pqu: hardware pq update descriptor
0168  * @raw: hardware raw (un-typed) descriptor
0169  * @txd: the generic software descriptor for all engines
0170  * @len: total transaction length for unmap
0171  * @result: asynchronous result of validate operations
0172  * @id: identifier for debug
0173  * @sed: pointer to super extended descriptor sw desc
0174  */
0175
0176 struct ioat_ring_ent {
0177     union {
0178         struct ioat_dma_descriptor *hw;
0179         struct ioat_xor_descriptor *xor;
0180         struct ioat_xor_ext_descriptor *xor_ex;
0181         struct ioat_pq_descriptor *pq;
0182         struct ioat_pq_ext_descriptor *pq_ex;
0183         struct ioat_pq_update_descriptor *pqu;
0184         struct ioat_raw_descriptor *raw;
0185     };
0186     size_t len;
0187     struct dma_async_tx_descriptor txd;
0188     enum sum_check_flags *result;
0189     #ifdef DEBUG
0190     int id;
0191     #endif
0192     struct ioat_sed_ent *sed;
0193 };
0194
0195 extern const struct sysfs_ops ioat_sysfs_ops;
0196 extern struct ioat_sysfs_entry ioat_version_attr;
0197 extern struct ioat_sysfs_entry ioat_cap_attr;
0198 extern int ioat_pending_level;
0199 extern int ioat_ring_alloc_order;
0200 extern struct kobj_type ioat_ktype;
0201 extern struct kmem_cache *ioat_cache;
0202 extern int ioat_ring_max_alloc_order;
0203 extern struct kmem_cache *ioat_sed_cache;
0204
0205 static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c)
0206 {
0207     return container_of(c, struct ioatdma_chan, dma_chan);
0208 }
0209
0210 /* wrapper around hardware descriptor format + additional software fields */
0211 #ifdef DEBUG
0212 #define set_desc_id(desc, i) ((desc)->id = (i))
0213 #define desc_id(desc) ((desc)->id)
0214 #else
0215 #define set_desc_id(desc, i)
0216 #define desc_id(desc) (0)
0217 #endif
0218
0219 static inline void
0220 __dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw,
0221         struct dma_async_tx_descriptor *tx, int id)
0222 {
0223     struct device *dev = to_dev(ioat_chan);
0224
0225     dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
0226         " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id,
0227         (unsigned long long) tx->phys,
0228         (unsigned long long) hw->next, tx->cookie, tx->flags,
0229         hw->ctl, hw->ctl_f.op, hw->ctl_f.int_en, hw->ctl_f.compl_write);
0230 }
0231
0232 #define dump_desc_dbg(c, d) \
0233     ({ if (d) __dump_desc_dbg(c, d->hw, &d->txd, desc_id(d)); 0; })
0234
0235 static inline struct ioatdma_chan *
0236 ioat_chan_by_index(struct ioatdma_device *ioat_dma, int index)
0237 {
0238     return ioat_dma->idx[index];
0239 }
0240
0241 static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan)
0242 {
0243     return readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET);
0244 }
0245
0246 static inline u64 ioat_chansts_to_addr(u64 status)
0247 {
0248     return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
0249 }
0250
0251 static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan)
0252 {
0253     return readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
0254 }
0255
0256 static inline void ioat_suspend(struct ioatdma_chan *ioat_chan)
0257 {
0258     u8 ver = ioat_chan->ioat_dma->version;
0259
0260     writeb(IOAT_CHANCMD_SUSPEND,
0261            ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
0262 }
0263
0264 static inline void ioat_reset(struct ioatdma_chan *ioat_chan)
0265 {
0266     u8 ver = ioat_chan->ioat_dma->version;
0267
0268     writeb(IOAT_CHANCMD_RESET,
0269            ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
0270 }
0271
0272 static inline bool ioat_reset_pending(struct ioatdma_chan *ioat_chan)
0273 {
0274     u8 ver = ioat_chan->ioat_dma->version;
0275     u8 cmd;
0276
0277     cmd = readb(ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
0278     return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET;
0279 }
0280
0281 static inline bool is_ioat_active(unsigned long status)
0282 {
0283     return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
0284 }
0285
0286 static inline bool is_ioat_idle(unsigned long status)
0287 {
0288     return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
0289 }
0290
0291 static inline bool is_ioat_halted(unsigned long status)
0292 {
0293     return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
0294 }
0295
0296 static inline bool is_ioat_suspended(unsigned long status)
0297 {
0298     return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
0299 }
0300
0301 /* channel was fatally programmed */
0302 static inline bool is_ioat_bug(unsigned long err)
0303 {
0304     return !!err;
0305 }
0306
0307
0308 static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan)
0309 {
0310     return 1 << ioat_chan->alloc_order;
0311 }
0312
0313 /* count of descriptors in flight with the engine */
0314 static inline u16 ioat_ring_active(struct ioatdma_chan *ioat_chan)
0315 {
0316     return CIRC_CNT(ioat_chan->head, ioat_chan->tail,
0317             ioat_ring_size(ioat_chan));
0318 }
0319
0320 /* count of descriptors pending submission to hardware */
0321 static inline u16 ioat_ring_pending(struct ioatdma_chan *ioat_chan)
0322 {
0323     return CIRC_CNT(ioat_chan->head, ioat_chan->issued,
0324             ioat_ring_size(ioat_chan));
0325 }
0326
0327 static inline u32 ioat_ring_space(struct ioatdma_chan *ioat_chan)
0328 {
0329     return ioat_ring_size(ioat_chan) - ioat_ring_active(ioat_chan);
0330 }
0331
0332 static inline u16
0333 ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len)
0334 {
0335     u16 num_descs = len >> ioat_chan->xfercap_log;
0336
0337     num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1));
0338     return num_descs;
0339 }
0340
0341 static inline struct ioat_ring_ent *
0342 ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx)
0343 {
0344     return ioat_chan->ring[idx & (ioat_ring_size(ioat_chan) - 1)];
0345 }
0346
0347 static inline void
0348 ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr)
0349 {
0350     writel(addr & 0x00000000FFFFFFFF,
0351            ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
0352     writel(addr >> 32,
0353            ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
0354 }
0355
0356 /* IOAT Prep functions */
0357 struct dma_async_tx_descriptor *
0358 ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
0359                dma_addr_t dma_src, size_t len, unsigned long flags);
0360 struct dma_async_tx_descriptor *
0361 ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags);
0362 struct dma_async_tx_descriptor *
0363 ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
0364            unsigned int src_cnt, size_t len, unsigned long flags);
0365 struct dma_async_tx_descriptor *
0366 ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
0367             unsigned int src_cnt, size_t len,
0368             enum sum_check_flags *result, unsigned long flags);
0369 struct dma_async_tx_descriptor *
0370 ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
0371           unsigned int src_cnt, const unsigned char *scf, size_t len,
0372           unsigned long flags);
0373 struct dma_async_tx_descriptor *
0374 ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
0375           unsigned int src_cnt, const unsigned char *scf, size_t len,
0376           enum sum_check_flags *pqres, unsigned long flags);
0377 struct dma_async_tx_descriptor *
0378 ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
0379          unsigned int src_cnt, size_t len, unsigned long flags);
0380 struct dma_async_tx_descriptor *
0381 ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
0382              unsigned int src_cnt, size_t len,
0383              enum sum_check_flags *result, unsigned long flags);
0384
0385 /* IOAT Operation functions */
0386 irqreturn_t ioat_dma_do_interrupt(int irq, void *data);
0387 irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data);
0388 struct ioat_ring_ent **
0389 ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags);
0390 void ioat_start_null_desc(struct ioatdma_chan *ioat_chan);
0391 void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan);
0392 int ioat_reset_hw(struct ioatdma_chan *ioat_chan);
0393 enum dma_status
0394 ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
0395         struct dma_tx_state *txstate);
0396 void ioat_cleanup_event(struct tasklet_struct *t);
0397 void ioat_timer_event(struct timer_list *t);
0398 int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs);
0399 void ioat_issue_pending(struct dma_chan *chan);
0400
0401 /* IOAT Init functions */
0402 bool is_bwd_ioat(struct pci_dev *pdev);
0403 struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
0404 void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type);
0405 void ioat_kobject_del(struct ioatdma_device *ioat_dma);
0406 int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma);
0407 void ioat_stop(struct ioatdma_chan *ioat_chan);
0408 #endif /* IOATDMA_H */