0001
0002
0003 #ifndef _IDXD_H_
0004 #define _IDXD_H_
0005
0006 #include <linux/sbitmap.h>
0007 #include <linux/dmaengine.h>
0008 #include <linux/percpu-rwsem.h>
0009 #include <linux/wait.h>
0010 #include <linux/cdev.h>
0011 #include <linux/idr.h>
0012 #include <linux/pci.h>
0013 #include <linux/ioasid.h>
0014 #include <linux/perf_event.h>
0015 #include <uapi/linux/idxd.h>
0016 #include "registers.h"
0017
0018 #define IDXD_DRIVER_VERSION "1.00"
0019
0020 extern struct kmem_cache *idxd_desc_pool;
0021 extern bool tc_override;
0022
0023 struct idxd_wq;
0024 struct idxd_dev;
0025
0026 enum idxd_dev_type {
0027 IDXD_DEV_NONE = -1,
0028 IDXD_DEV_DSA = 0,
0029 IDXD_DEV_IAX,
0030 IDXD_DEV_WQ,
0031 IDXD_DEV_GROUP,
0032 IDXD_DEV_ENGINE,
0033 IDXD_DEV_CDEV,
0034 IDXD_DEV_MAX_TYPE,
0035 };
0036
0037 struct idxd_dev {
0038 struct device conf_dev;
0039 enum idxd_dev_type type;
0040 };
0041
0042 #define IDXD_REG_TIMEOUT 50
0043 #define IDXD_DRAIN_TIMEOUT 5000
0044
0045 enum idxd_type {
0046 IDXD_TYPE_UNKNOWN = -1,
0047 IDXD_TYPE_DSA = 0,
0048 IDXD_TYPE_IAX,
0049 IDXD_TYPE_MAX,
0050 };
0051
0052 #define IDXD_NAME_SIZE 128
0053 #define IDXD_PMU_EVENT_MAX 64
0054
0055 #define IDXD_ENQCMDS_RETRIES 32
0056 #define IDXD_ENQCMDS_MAX_RETRIES 64
0057
0058 struct idxd_device_driver {
0059 const char *name;
0060 enum idxd_dev_type *type;
0061 int (*probe)(struct idxd_dev *idxd_dev);
0062 void (*remove)(struct idxd_dev *idxd_dev);
0063 struct device_driver drv;
0064 };
0065
0066 extern struct idxd_device_driver dsa_drv;
0067 extern struct idxd_device_driver idxd_drv;
0068 extern struct idxd_device_driver idxd_dmaengine_drv;
0069 extern struct idxd_device_driver idxd_user_drv;
0070
0071 #define INVALID_INT_HANDLE -1
0072 struct idxd_irq_entry {
0073 int id;
0074 int vector;
0075 struct llist_head pending_llist;
0076 struct list_head work_list;
0077
0078
0079
0080
0081 spinlock_t list_lock;
0082 int int_handle;
0083 ioasid_t pasid;
0084 };
0085
0086 struct idxd_group {
0087 struct idxd_dev idxd_dev;
0088 struct idxd_device *idxd;
0089 struct grpcfg grpcfg;
0090 int id;
0091 int num_engines;
0092 int num_wqs;
0093 bool use_rdbuf_limit;
0094 u8 rdbufs_allowed;
0095 u8 rdbufs_reserved;
0096 int tc_a;
0097 int tc_b;
0098 };
0099
0100 struct idxd_pmu {
0101 struct idxd_device *idxd;
0102
0103 struct perf_event *event_list[IDXD_PMU_EVENT_MAX];
0104 int n_events;
0105
0106 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX);
0107
0108 struct pmu pmu;
0109 char name[IDXD_NAME_SIZE];
0110 int cpu;
0111
0112 int n_counters;
0113 int counter_width;
0114 int n_event_categories;
0115
0116 bool per_counter_caps_supported;
0117 unsigned long supported_event_categories;
0118
0119 unsigned long supported_filters;
0120 int n_filters;
0121
0122 struct hlist_node cpuhp_node;
0123 };
0124
0125 #define IDXD_MAX_PRIORITY 0xf
0126
0127 enum idxd_wq_state {
0128 IDXD_WQ_DISABLED = 0,
0129 IDXD_WQ_ENABLED,
0130 };
0131
0132 enum idxd_wq_flag {
0133 WQ_FLAG_DEDICATED = 0,
0134 WQ_FLAG_BLOCK_ON_FAULT,
0135 };
0136
0137 enum idxd_wq_type {
0138 IDXD_WQT_NONE = 0,
0139 IDXD_WQT_KERNEL,
0140 IDXD_WQT_USER,
0141 };
0142
0143 struct idxd_cdev {
0144 struct idxd_wq *wq;
0145 struct cdev cdev;
0146 struct idxd_dev idxd_dev;
0147 int minor;
0148 };
0149
0150 #define IDXD_ALLOCATED_BATCH_SIZE 128U
0151 #define WQ_NAME_SIZE 1024
0152 #define WQ_TYPE_SIZE 10
0153
0154 #define WQ_DEFAULT_QUEUE_DEPTH 16
0155 #define WQ_DEFAULT_MAX_XFER SZ_2M
0156 #define WQ_DEFAULT_MAX_BATCH 32
0157
0158 enum idxd_op_type {
0159 IDXD_OP_BLOCK = 0,
0160 IDXD_OP_NONBLOCK = 1,
0161 };
0162
0163 enum idxd_complete_type {
0164 IDXD_COMPLETE_NORMAL = 0,
0165 IDXD_COMPLETE_ABORT,
0166 IDXD_COMPLETE_DEV_FAIL,
0167 };
0168
0169 struct idxd_dma_chan {
0170 struct dma_chan chan;
0171 struct idxd_wq *wq;
0172 };
0173
0174 struct idxd_wq {
0175 void __iomem *portal;
0176 u32 portal_offset;
0177 unsigned int enqcmds_retries;
0178 struct percpu_ref wq_active;
0179 struct completion wq_dead;
0180 struct completion wq_resurrect;
0181 struct idxd_dev idxd_dev;
0182 struct idxd_cdev *idxd_cdev;
0183 struct wait_queue_head err_queue;
0184 struct idxd_device *idxd;
0185 int id;
0186 struct idxd_irq_entry ie;
0187 enum idxd_wq_type type;
0188 struct idxd_group *group;
0189 int client_count;
0190 struct mutex wq_lock;
0191 u32 size;
0192 u32 threshold;
0193 u32 priority;
0194 enum idxd_wq_state state;
0195 unsigned long flags;
0196 union wqcfg *wqcfg;
0197 struct dsa_hw_desc **hw_descs;
0198 int num_descs;
0199 union {
0200 struct dsa_completion_record *compls;
0201 struct iax_completion_record *iax_compls;
0202 };
0203 dma_addr_t compls_addr;
0204 int compls_size;
0205 struct idxd_desc **descs;
0206 struct sbitmap_queue sbq;
0207 struct idxd_dma_chan *idxd_chan;
0208 char name[WQ_NAME_SIZE + 1];
0209 u64 max_xfer_bytes;
0210 u32 max_batch_size;
0211 bool ats_dis;
0212 };
0213
0214 struct idxd_engine {
0215 struct idxd_dev idxd_dev;
0216 int id;
0217 struct idxd_group *group;
0218 struct idxd_device *idxd;
0219 };
0220
0221
0222 struct idxd_hw {
0223 u32 version;
0224 union gen_cap_reg gen_cap;
0225 union wq_cap_reg wq_cap;
0226 union group_cap_reg group_cap;
0227 union engine_cap_reg engine_cap;
0228 struct opcap opcap;
0229 u32 cmd_cap;
0230 };
0231
0232 enum idxd_device_state {
0233 IDXD_DEV_HALTED = -1,
0234 IDXD_DEV_DISABLED = 0,
0235 IDXD_DEV_ENABLED,
0236 };
0237
0238 enum idxd_device_flag {
0239 IDXD_FLAG_CONFIGURABLE = 0,
0240 IDXD_FLAG_CMD_RUNNING,
0241 IDXD_FLAG_PASID_ENABLED,
0242 IDXD_FLAG_USER_PASID_ENABLED,
0243 };
0244
0245 struct idxd_dma_dev {
0246 struct idxd_device *idxd;
0247 struct dma_device dma;
0248 };
0249
0250 struct idxd_driver_data {
0251 const char *name_prefix;
0252 enum idxd_type type;
0253 struct device_type *dev_type;
0254 int compl_size;
0255 int align;
0256 };
0257
0258 struct idxd_device {
0259 struct idxd_dev idxd_dev;
0260 struct idxd_driver_data *data;
0261 struct list_head list;
0262 struct idxd_hw hw;
0263 enum idxd_device_state state;
0264 unsigned long flags;
0265 int id;
0266 int major;
0267 u32 cmd_status;
0268 struct idxd_irq_entry ie;
0269
0270 struct pci_dev *pdev;
0271 void __iomem *reg_base;
0272
0273 spinlock_t dev_lock;
0274 spinlock_t cmd_lock;
0275 struct completion *cmd_done;
0276 struct idxd_group **groups;
0277 struct idxd_wq **wqs;
0278 struct idxd_engine **engines;
0279
0280 struct iommu_sva *sva;
0281 unsigned int pasid;
0282
0283 int num_groups;
0284 int irq_cnt;
0285 bool request_int_handles;
0286
0287 u32 msix_perm_offset;
0288 u32 wqcfg_offset;
0289 u32 grpcfg_offset;
0290 u32 perfmon_offset;
0291
0292 u64 max_xfer_bytes;
0293 u32 max_batch_size;
0294 int max_groups;
0295 int max_engines;
0296 int max_rdbufs;
0297 int max_wqs;
0298 int max_wq_size;
0299 int rdbuf_limit;
0300 int nr_rdbufs;
0301 unsigned int wqcfg_size;
0302
0303 union sw_err_reg sw_err;
0304 wait_queue_head_t cmd_waitq;
0305
0306 struct idxd_dma_dev *idxd_dma;
0307 struct workqueue_struct *wq;
0308 struct work_struct work;
0309
0310 struct idxd_pmu *idxd_pmu;
0311 };
0312
0313
0314 struct idxd_desc {
0315 union {
0316 struct dsa_hw_desc *hw;
0317 struct iax_hw_desc *iax_hw;
0318 };
0319 dma_addr_t desc_dma;
0320 union {
0321 struct dsa_completion_record *completion;
0322 struct iax_completion_record *iax_completion;
0323 };
0324 dma_addr_t compl_dma;
0325 struct dma_async_tx_descriptor txd;
0326 struct llist_node llnode;
0327 struct list_head list;
0328 int id;
0329 int cpu;
0330 struct idxd_wq *wq;
0331 };
0332
0333
0334
0335
0336
0337 enum idxd_completion_status {
0338 IDXD_COMP_DESC_ABORT = 0xff,
0339 };
0340
0341 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev
0342 #define wq_confdev(wq) &wq->idxd_dev.conf_dev
0343 #define engine_confdev(engine) &engine->idxd_dev.conf_dev
0344 #define group_confdev(group) &group->idxd_dev.conf_dev
0345 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev
0346
0347 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev)
0348 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev)
0349 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev)
0350
0351 static inline struct idxd_device *confdev_to_idxd(struct device *dev)
0352 {
0353 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
0354
0355 return idxd_dev_to_idxd(idxd_dev);
0356 }
0357
0358 static inline struct idxd_wq *confdev_to_wq(struct device *dev)
0359 {
0360 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
0361
0362 return idxd_dev_to_wq(idxd_dev);
0363 }
0364
0365 static inline struct idxd_engine *confdev_to_engine(struct device *dev)
0366 {
0367 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
0368
0369 return container_of(idxd_dev, struct idxd_engine, idxd_dev);
0370 }
0371
0372 static inline struct idxd_group *confdev_to_group(struct device *dev)
0373 {
0374 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
0375
0376 return container_of(idxd_dev, struct idxd_group, idxd_dev);
0377 }
0378
0379 static inline struct idxd_cdev *dev_to_cdev(struct device *dev)
0380 {
0381 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
0382
0383 return container_of(idxd_dev, struct idxd_cdev, idxd_dev);
0384 }
0385
0386 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type)
0387 {
0388 if (type >= IDXD_DEV_MAX_TYPE) {
0389 idev->type = IDXD_DEV_NONE;
0390 return;
0391 }
0392
0393 idev->type = type;
0394 }
0395
0396 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx)
0397 {
0398 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie;
0399 }
0400
0401 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie)
0402 {
0403 return container_of(ie, struct idxd_wq, ie);
0404 }
0405
0406 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie)
0407 {
0408 return container_of(ie, struct idxd_device, ie);
0409 }
0410
0411 extern struct bus_type dsa_bus_type;
0412
0413 extern bool support_enqcmd;
0414 extern struct ida idxd_ida;
0415 extern struct device_type dsa_device_type;
0416 extern struct device_type iax_device_type;
0417 extern struct device_type idxd_wq_device_type;
0418 extern struct device_type idxd_engine_device_type;
0419 extern struct device_type idxd_group_device_type;
0420
0421 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev)
0422 {
0423 return idxd_dev->type == IDXD_DEV_DSA;
0424 }
0425
0426 static inline bool is_iax_dev(struct idxd_dev *idxd_dev)
0427 {
0428 return idxd_dev->type == IDXD_DEV_IAX;
0429 }
0430
0431 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev)
0432 {
0433 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev);
0434 }
0435
0436 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev)
0437 {
0438 return idxd_dev->type == IDXD_DEV_WQ;
0439 }
0440
0441 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
0442 {
0443 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
0444 return true;
0445 return false;
0446 }
0447
0448 static inline bool is_idxd_wq_user(struct idxd_wq *wq)
0449 {
0450 return wq->type == IDXD_WQT_USER;
0451 }
0452
0453 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq)
0454 {
0455 return wq->type == IDXD_WQT_KERNEL;
0456 }
0457
0458 static inline bool wq_dedicated(struct idxd_wq *wq)
0459 {
0460 return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
0461 }
0462
0463 static inline bool wq_shared(struct idxd_wq *wq)
0464 {
0465 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
0466 }
0467
0468 static inline bool device_pasid_enabled(struct idxd_device *idxd)
0469 {
0470 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
0471 }
0472
0473 static inline bool device_user_pasid_enabled(struct idxd_device *idxd)
0474 {
0475 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
0476 }
0477
0478 static inline bool wq_pasid_enabled(struct idxd_wq *wq)
0479 {
0480 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||
0481 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));
0482 }
0483
0484 static inline bool wq_shared_supported(struct idxd_wq *wq)
0485 {
0486 return (support_enqcmd && wq_pasid_enabled(wq));
0487 }
0488
0489 enum idxd_portal_prot {
0490 IDXD_PORTAL_UNLIMITED = 0,
0491 IDXD_PORTAL_LIMITED,
0492 };
0493
0494 enum idxd_interrupt_type {
0495 IDXD_IRQ_MSIX = 0,
0496 IDXD_IRQ_IMS,
0497 };
0498
0499 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
0500 {
0501 return prot * 0x1000;
0502 }
0503
0504 static inline int idxd_get_wq_portal_full_offset(int wq_id,
0505 enum idxd_portal_prot prot)
0506 {
0507 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
0508 }
0509
0510 #define IDXD_PORTAL_MASK (PAGE_SIZE - 1)
0511
0512
0513
0514
0515
0516
0517
0518
0519
0520 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq)
0521 {
0522 int ofs = wq->portal_offset;
0523
0524 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK;
0525 return wq->portal + ofs;
0526 }
0527
0528 static inline void idxd_wq_get(struct idxd_wq *wq)
0529 {
0530 wq->client_count++;
0531 }
0532
0533 static inline void idxd_wq_put(struct idxd_wq *wq)
0534 {
0535 wq->client_count--;
0536 }
0537
0538 static inline int idxd_wq_refcount(struct idxd_wq *wq)
0539 {
0540 return wq->client_count;
0541 };
0542
0543 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv,
0544 struct module *module, const char *mod_name);
0545 #define idxd_driver_register(driver) \
0546 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
0547
0548 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv);
0549
0550 #define module_idxd_driver(__idxd_driver) \
0551 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister)
0552
0553 int idxd_register_bus_type(void);
0554 void idxd_unregister_bus_type(void);
0555 int idxd_register_devices(struct idxd_device *idxd);
0556 void idxd_unregister_devices(struct idxd_device *idxd);
0557 int idxd_register_driver(void);
0558 void idxd_unregister_driver(void);
0559 void idxd_wqs_quiesce(struct idxd_device *idxd);
0560 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc);
0561
0562
0563 irqreturn_t idxd_misc_thread(int vec, void *data);
0564 irqreturn_t idxd_wq_thread(int irq, void *data);
0565 void idxd_mask_error_interrupts(struct idxd_device *idxd);
0566 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
0567
0568
0569 int idxd_register_idxd_drv(void);
0570 void idxd_unregister_idxd_drv(void);
0571 int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
0572 void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
0573 int drv_enable_wq(struct idxd_wq *wq);
0574 void drv_disable_wq(struct idxd_wq *wq);
0575 int idxd_device_init_reset(struct idxd_device *idxd);
0576 int idxd_device_enable(struct idxd_device *idxd);
0577 int idxd_device_disable(struct idxd_device *idxd);
0578 void idxd_device_reset(struct idxd_device *idxd);
0579 void idxd_device_clear_state(struct idxd_device *idxd);
0580 int idxd_device_config(struct idxd_device *idxd);
0581 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
0582 int idxd_device_load_config(struct idxd_device *idxd);
0583 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
0584 enum idxd_interrupt_type irq_type);
0585 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
0586 enum idxd_interrupt_type irq_type);
0587
0588
0589 void idxd_wqs_unmap_portal(struct idxd_device *idxd);
0590 int idxd_wq_alloc_resources(struct idxd_wq *wq);
0591 void idxd_wq_free_resources(struct idxd_wq *wq);
0592 int idxd_wq_enable(struct idxd_wq *wq);
0593 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config);
0594 void idxd_wq_drain(struct idxd_wq *wq);
0595 void idxd_wq_reset(struct idxd_wq *wq);
0596 int idxd_wq_map_portal(struct idxd_wq *wq);
0597 void idxd_wq_unmap_portal(struct idxd_wq *wq);
0598 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
0599 int idxd_wq_disable_pasid(struct idxd_wq *wq);
0600 void __idxd_wq_quiesce(struct idxd_wq *wq);
0601 void idxd_wq_quiesce(struct idxd_wq *wq);
0602 int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
0603 void idxd_wq_free_irq(struct idxd_wq *wq);
0604 int idxd_wq_request_irq(struct idxd_wq *wq);
0605
0606
0607 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
0608 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
0609 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
0610 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);
0611
0612
0613 int idxd_register_dma_device(struct idxd_device *idxd);
0614 void idxd_unregister_dma_device(struct idxd_device *idxd);
0615 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
0616 void idxd_dma_complete_txd(struct idxd_desc *desc,
0617 enum idxd_complete_type comp_type, bool free_desc);
0618
0619
0620 int idxd_cdev_register(void);
0621 void idxd_cdev_remove(void);
0622 int idxd_cdev_get_major(struct idxd_device *idxd);
0623 int idxd_wq_add_cdev(struct idxd_wq *wq);
0624 void idxd_wq_del_cdev(struct idxd_wq *wq);
0625
0626
0627 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)
0628 int perfmon_pmu_init(struct idxd_device *idxd);
0629 void perfmon_pmu_remove(struct idxd_device *idxd);
0630 void perfmon_counter_overflow(struct idxd_device *idxd);
0631 void perfmon_init(void);
0632 void perfmon_exit(void);
0633 #else
0634 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; }
0635 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {}
0636 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {}
0637 static inline void perfmon_init(void) {}
0638 static inline void perfmon_exit(void) {}
0639 #endif
0640
0641 #endif