include/linux/msi.h

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 #ifndef LINUX_MSI_H
0003 #define LINUX_MSI_H
0004
0005 /*
0006  * This header file contains MSI data structures and functions which are
0007  * only relevant for:
0008  *  - Interrupt core code
0009  *  - PCI/MSI core code
0010  *  - MSI interrupt domain implementations
0011  *  - IOMMU, low level VFIO, NTB and other justified exceptions
0012  *    dealing with low level MSI details.
0013  *
0014  * Regular device drivers have no business with any of these functions and
0015  * especially storing MSI descriptor pointers in random code is considered
0016  * abuse. The only function which is relevant for drivers is msi_get_virq().
0017  */
0018
0019 #include <linux/cpumask.h>
0020 #include <linux/xarray.h>
0021 #include <linux/mutex.h>
0022 #include <linux/list.h>
0023 #include <asm/msi.h>
0024
0025 /* Dummy shadow structures if an architecture does not define them */
0026 #ifndef arch_msi_msg_addr_lo
0027 typedef struct arch_msi_msg_addr_lo {
0028     u32 address_lo;
0029 } __attribute__ ((packed)) arch_msi_msg_addr_lo_t;
0030 #endif
0031
0032 #ifndef arch_msi_msg_addr_hi
0033 typedef struct arch_msi_msg_addr_hi {
0034     u32 address_hi;
0035 } __attribute__ ((packed)) arch_msi_msg_addr_hi_t;
0036 #endif
0037
0038 #ifndef arch_msi_msg_data
0039 typedef struct arch_msi_msg_data {
0040     u32 data;
0041 } __attribute__ ((packed)) arch_msi_msg_data_t;
0042 #endif
0043
0044 /**
0045  * msi_msg - Representation of a MSI message
0046  * @address_lo:     Low 32 bits of msi message address
0047  * @arch_addrlo:    Architecture specific shadow of @address_lo
0048  * @address_hi:     High 32 bits of msi message address
0049  *          (only used when device supports it)
0050  * @arch_addrhi:    Architecture specific shadow of @address_hi
0051  * @data:       MSI message data (usually 16 bits)
0052  * @arch_data:      Architecture specific shadow of @data
0053  */
0054 struct msi_msg {
0055     union {
0056         u32         address_lo;
0057         arch_msi_msg_addr_lo_t  arch_addr_lo;
0058     };
0059     union {
0060         u32         address_hi;
0061         arch_msi_msg_addr_hi_t  arch_addr_hi;
0062     };
0063     union {
0064         u32         data;
0065         arch_msi_msg_data_t arch_data;
0066     };
0067 };
0068
0069 extern int pci_msi_ignore_mask;
0070 /* Helper functions */
0071 struct irq_data;
0072 struct msi_desc;
0073 struct pci_dev;
0074 struct platform_msi_priv_data;
0075 struct device_attribute;
0076
0077 void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg);
0078 #ifdef CONFIG_GENERIC_MSI_IRQ
0079 void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg);
0080 #else
0081 static inline void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg)
0082 {
0083 }
0084 #endif
0085
0086 typedef void (*irq_write_msi_msg_t)(struct msi_desc *desc,
0087                     struct msi_msg *msg);
0088
0089 /**
0090  * pci_msi_desc - PCI/MSI specific MSI descriptor data
0091  *
0092  * @msi_mask:   [PCI MSI]   MSI cached mask bits
0093  * @msix_ctrl:  [PCI MSI-X] MSI-X cached per vector control bits
0094  * @is_msix:    [PCI MSI/X] True if MSI-X
0095  * @multiple:   [PCI MSI/X] log2 num of messages allocated
0096  * @multi_cap:  [PCI MSI/X] log2 num of messages supported
0097  * @can_mask:   [PCI MSI/X] Masking supported?
0098  * @is_64:  [PCI MSI/X] Address size: 0=32bit 1=64bit
0099  * @default_irq:[PCI MSI/X] The default pre-assigned non-MSI irq
0100  * @mask_pos:   [PCI MSI]   Mask register position
0101  * @mask_base:  [PCI MSI-X] Mask register base address
0102  */
0103 struct pci_msi_desc {
0104     union {
0105         u32 msi_mask;
0106         u32 msix_ctrl;
0107     };
0108     struct {
0109         u8  is_msix     : 1;
0110         u8  multiple    : 3;
0111         u8  multi_cap   : 3;
0112         u8  can_mask    : 1;
0113         u8  is_64       : 1;
0114         u8  is_virtual  : 1;
0115         unsigned default_irq;
0116     } msi_attrib;
0117     union {
0118         u8  mask_pos;
0119         void __iomem *mask_base;
0120     };
0121 };
0122
0123 #define MSI_MAX_INDEX       ((unsigned int)USHRT_MAX)
0124
0125 /**
0126  * struct msi_desc - Descriptor structure for MSI based interrupts
0127  * @irq:    The base interrupt number
0128  * @nvec_used:  The number of vectors used
0129  * @dev:    Pointer to the device which uses this descriptor
0130  * @msg:    The last set MSI message cached for reuse
0131  * @affinity:   Optional pointer to a cpu affinity mask for this descriptor
0132  * @sysfs_attr: Pointer to sysfs device attribute
0133  *
0134  * @write_msi_msg:  Callback that may be called when the MSI message
0135  *          address or data changes
0136  * @write_msi_msg_data: Data parameter for the callback.
0137  *
0138  * @msi_index:  Index of the msi descriptor
0139  * @pci:    PCI specific msi descriptor data
0140  */
0141 struct msi_desc {
0142     /* Shared device/bus type independent data */
0143     unsigned int            irq;
0144     unsigned int            nvec_used;
0145     struct device           *dev;
0146     struct msi_msg          msg;
0147     struct irq_affinity_desc    *affinity;
0148 #ifdef CONFIG_IRQ_MSI_IOMMU
0149     const void          *iommu_cookie;
0150 #endif
0151 #ifdef CONFIG_SYSFS
0152     struct device_attribute     *sysfs_attrs;
0153 #endif
0154
0155     void (*write_msi_msg)(struct msi_desc *entry, void *data);
0156     void *write_msi_msg_data;
0157
0158     u16             msi_index;
0159     struct pci_msi_desc     pci;
0160 };
0161
0162 /*
0163  * Filter values for the MSI descriptor iterators and accessor functions.
0164  */
0165 enum msi_desc_filter {
0166     /* All descriptors */
0167     MSI_DESC_ALL,
0168     /* Descriptors which have no interrupt associated */
0169     MSI_DESC_NOTASSOCIATED,
0170     /* Descriptors which have an interrupt associated */
0171     MSI_DESC_ASSOCIATED,
0172 };
0173
0174 /**
0175  * msi_device_data - MSI per device data
0176  * @properties:     MSI properties which are interesting to drivers
0177  * @platform_data:  Platform-MSI specific data
0178  * @mutex:      Mutex protecting the MSI descriptor store
0179  * @__store:        Xarray for storing MSI descriptor pointers
0180  * @__iter_idx:     Index to search the next entry for iterators
0181  */
0182 struct msi_device_data {
0183     unsigned long           properties;
0184     struct platform_msi_priv_data   *platform_data;
0185     struct mutex            mutex;
0186     struct xarray           __store;
0187     unsigned long           __iter_idx;
0188 };
0189
0190 int msi_setup_device_data(struct device *dev);
0191
0192 unsigned int msi_get_virq(struct device *dev, unsigned int index);
0193 void msi_lock_descs(struct device *dev);
0194 void msi_unlock_descs(struct device *dev);
0195
0196 struct msi_desc *msi_first_desc(struct device *dev, enum msi_desc_filter filter);
0197 struct msi_desc *msi_next_desc(struct device *dev, enum msi_desc_filter filter);
0198
0199 /**
0200  * msi_for_each_desc - Iterate the MSI descriptors
0201  *
0202  * @desc:   struct msi_desc pointer used as iterator
0203  * @dev:    struct device pointer - device to iterate
0204  * @filter: Filter for descriptor selection
0205  *
0206  * Notes:
0207  *  - The loop must be protected with a msi_lock_descs()/msi_unlock_descs()
0208  *    pair.
0209  *  - It is safe to remove a retrieved MSI descriptor in the loop.
0210  */
0211 #define msi_for_each_desc(desc, dev, filter)            \
0212     for ((desc) = msi_first_desc((dev), (filter)); (desc);  \
0213          (desc) = msi_next_desc((dev), (filter)))
0214
0215 #define msi_desc_to_dev(desc)       ((desc)->dev)
0216
0217 #ifdef CONFIG_IRQ_MSI_IOMMU
0218 static inline const void *msi_desc_get_iommu_cookie(struct msi_desc *desc)
0219 {
0220     return desc->iommu_cookie;
0221 }
0222
0223 static inline void msi_desc_set_iommu_cookie(struct msi_desc *desc,
0224                          const void *iommu_cookie)
0225 {
0226     desc->iommu_cookie = iommu_cookie;
0227 }
0228 #else
0229 static inline const void *msi_desc_get_iommu_cookie(struct msi_desc *desc)
0230 {
0231     return NULL;
0232 }
0233
0234 static inline void msi_desc_set_iommu_cookie(struct msi_desc *desc,
0235                          const void *iommu_cookie)
0236 {
0237 }
0238 #endif
0239
0240 #ifdef CONFIG_PCI_MSI
0241 struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc);
0242 void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg);
0243 #else /* CONFIG_PCI_MSI */
0244 static inline void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg)
0245 {
0246 }
0247 #endif /* CONFIG_PCI_MSI */
0248
0249 int msi_add_msi_desc(struct device *dev, struct msi_desc *init_desc);
0250 void msi_free_msi_descs_range(struct device *dev, enum msi_desc_filter filter,
0251                   unsigned int first_index, unsigned int last_index);
0252
0253 /**
0254  * msi_free_msi_descs - Free MSI descriptors of a device
0255  * @dev:    Device to free the descriptors
0256  */
0257 static inline void msi_free_msi_descs(struct device *dev)
0258 {
0259     msi_free_msi_descs_range(dev, MSI_DESC_ALL, 0, MSI_MAX_INDEX);
0260 }
0261
0262 void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg);
0263 void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg);
0264
0265 void pci_msi_mask_irq(struct irq_data *data);
0266 void pci_msi_unmask_irq(struct irq_data *data);
0267
0268 /*
0269  * The arch hooks to setup up msi irqs. Default functions are implemented
0270  * as weak symbols so that they /can/ be overriden by architecture specific
0271  * code if needed. These hooks can only be enabled by the architecture.
0272  *
0273  * If CONFIG_PCI_MSI_ARCH_FALLBACKS is not selected they are replaced by
0274  * stubs with warnings.
0275  */
0276 #ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS
0277 int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc);
0278 void arch_teardown_msi_irq(unsigned int irq);
0279 int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type);
0280 void arch_teardown_msi_irqs(struct pci_dev *dev);
0281 #ifdef CONFIG_SYSFS
0282 int msi_device_populate_sysfs(struct device *dev);
0283 void msi_device_destroy_sysfs(struct device *dev);
0284 #else /* CONFIG_SYSFS */
0285 static inline int msi_device_populate_sysfs(struct device *dev) { return 0; }
0286 static inline void msi_device_destroy_sysfs(struct device *dev) { }
0287 #endif /* !CONFIG_SYSFS */
0288 #endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */
0289
0290 /*
0291  * The restore hook is still available even for fully irq domain based
0292  * setups. Courtesy to XEN/X86.
0293  */
0294 bool arch_restore_msi_irqs(struct pci_dev *dev);
0295
0296 #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
0297
0298 #include <linux/irqhandler.h>
0299
0300 struct irq_domain;
0301 struct irq_domain_ops;
0302 struct irq_chip;
0303 struct device_node;
0304 struct fwnode_handle;
0305 struct msi_domain_info;
0306
0307 /**
0308  * struct msi_domain_ops - MSI interrupt domain callbacks
0309  * @get_hwirq:      Retrieve the resulting hw irq number
0310  * @msi_init:       Domain specific init function for MSI interrupts
0311  * @msi_free:       Domain specific function to free a MSI interrupts
0312  * @msi_check:      Callback for verification of the domain/info/dev data
0313  * @msi_prepare:    Prepare the allocation of the interrupts in the domain
0314  * @set_desc:       Set the msi descriptor for an interrupt
0315  * @domain_alloc_irqs:  Optional function to override the default allocation
0316  *          function.
0317  * @domain_free_irqs:   Optional function to override the default free
0318  *          function.
0319  *
0320  * @get_hwirq, @msi_init and @msi_free are callbacks used by the underlying
0321  * irqdomain.
0322  *
0323  * @msi_check, @msi_prepare and @set_desc are callbacks used by
0324  * msi_domain_alloc/free_irqs().
0325  *
0326  * @domain_alloc_irqs, @domain_free_irqs can be used to override the
0327  * default allocation/free functions (__msi_domain_alloc/free_irqs). This
0328  * is initially for a wrapper around XENs seperate MSI universe which can't
0329  * be wrapped into the regular irq domains concepts by mere mortals.  This
0330  * allows to universally use msi_domain_alloc/free_irqs without having to
0331  * special case XEN all over the place.
0332  *
0333  * Contrary to other operations @domain_alloc_irqs and @domain_free_irqs
0334  * are set to the default implementation if NULL and even when
0335  * MSI_FLAG_USE_DEF_DOM_OPS is not set to avoid breaking existing users and
0336  * because these callbacks are obviously mandatory.
0337  *
0338  * This is NOT meant to be abused, but it can be useful to build wrappers
0339  * for specialized MSI irq domains which need extra work before and after
0340  * calling __msi_domain_alloc_irqs()/__msi_domain_free_irqs().
0341  */
0342 struct msi_domain_ops {
0343     irq_hw_number_t (*get_hwirq)(struct msi_domain_info *info,
0344                      msi_alloc_info_t *arg);
0345     int     (*msi_init)(struct irq_domain *domain,
0346                     struct msi_domain_info *info,
0347                     unsigned int virq, irq_hw_number_t hwirq,
0348                     msi_alloc_info_t *arg);
0349     void        (*msi_free)(struct irq_domain *domain,
0350                     struct msi_domain_info *info,
0351                     unsigned int virq);
0352     int     (*msi_check)(struct irq_domain *domain,
0353                      struct msi_domain_info *info,
0354                      struct device *dev);
0355     int     (*msi_prepare)(struct irq_domain *domain,
0356                        struct device *dev, int nvec,
0357                        msi_alloc_info_t *arg);
0358     void        (*set_desc)(msi_alloc_info_t *arg,
0359                     struct msi_desc *desc);
0360     int     (*domain_alloc_irqs)(struct irq_domain *domain,
0361                          struct device *dev, int nvec);
0362     void        (*domain_free_irqs)(struct irq_domain *domain,
0363                         struct device *dev);
0364 };
0365
0366 /**
0367  * struct msi_domain_info - MSI interrupt domain data
0368  * @flags:      Flags to decribe features and capabilities
0369  * @ops:        The callback data structure
0370  * @chip:       Optional: associated interrupt chip
0371  * @chip_data:      Optional: associated interrupt chip data
0372  * @handler:        Optional: associated interrupt flow handler
0373  * @handler_data:   Optional: associated interrupt flow handler data
0374  * @handler_name:   Optional: associated interrupt flow handler name
0375  * @data:       Optional: domain specific data
0376  */
0377 struct msi_domain_info {
0378     u32         flags;
0379     struct msi_domain_ops   *ops;
0380     struct irq_chip     *chip;
0381     void            *chip_data;
0382     irq_flow_handler_t  handler;
0383     void            *handler_data;
0384     const char      *handler_name;
0385     void            *data;
0386 };
0387
0388 /* Flags for msi_domain_info */
0389 enum {
0390     /*
0391      * Init non implemented ops callbacks with default MSI domain
0392      * callbacks.
0393      */
0394     MSI_FLAG_USE_DEF_DOM_OPS    = (1 << 0),
0395     /*
0396      * Init non implemented chip callbacks with default MSI chip
0397      * callbacks.
0398      */
0399     MSI_FLAG_USE_DEF_CHIP_OPS   = (1 << 1),
0400     /* Support multiple PCI MSI interrupts */
0401     MSI_FLAG_MULTI_PCI_MSI      = (1 << 2),
0402     /* Support PCI MSIX interrupts */
0403     MSI_FLAG_PCI_MSIX       = (1 << 3),
0404     /* Needs early activate, required for PCI */
0405     MSI_FLAG_ACTIVATE_EARLY     = (1 << 4),
0406     /*
0407      * Must reactivate when irq is started even when
0408      * MSI_FLAG_ACTIVATE_EARLY has been set.
0409      */
0410     MSI_FLAG_MUST_REACTIVATE    = (1 << 5),
0411     /* Is level-triggered capable, using two messages */
0412     MSI_FLAG_LEVEL_CAPABLE      = (1 << 6),
0413     /* Populate sysfs on alloc() and destroy it on free() */
0414     MSI_FLAG_DEV_SYSFS      = (1 << 7),
0415     /* MSI-X entries must be contiguous */
0416     MSI_FLAG_MSIX_CONTIGUOUS    = (1 << 8),
0417     /* Allocate simple MSI descriptors */
0418     MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS = (1 << 9),
0419     /* Free MSI descriptors */
0420     MSI_FLAG_FREE_MSI_DESCS     = (1 << 10),
0421 };
0422
0423 int msi_domain_set_affinity(struct irq_data *data, const struct cpumask *mask,
0424                 bool force);
0425
0426 struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode,
0427                      struct msi_domain_info *info,
0428                      struct irq_domain *parent);
0429 int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
0430                 int nvec);
0431 int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device *dev,
0432                        int nvec);
0433 int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
0434               int nvec);
0435 void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev);
0436 void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device *dev);
0437 void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev);
0438 struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain);
0439
0440 struct irq_domain *platform_msi_create_irq_domain(struct fwnode_handle *fwnode,
0441                           struct msi_domain_info *info,
0442                           struct irq_domain *parent);
0443 int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec,
0444                    irq_write_msi_msg_t write_msi_msg);
0445 void platform_msi_domain_free_irqs(struct device *dev);
0446
0447 /* When an MSI domain is used as an intermediate domain */
0448 int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev,
0449                 int nvec, msi_alloc_info_t *args);
0450 int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev,
0451                  int virq, int nvec, msi_alloc_info_t *args);
0452 struct irq_domain *
0453 __platform_msi_create_device_domain(struct device *dev,
0454                     unsigned int nvec,
0455                     bool is_tree,
0456                     irq_write_msi_msg_t write_msi_msg,
0457                     const struct irq_domain_ops *ops,
0458                     void *host_data);
0459
0460 #define platform_msi_create_device_domain(dev, nvec, write, ops, data)  \
0461     __platform_msi_create_device_domain(dev, nvec, false, write, ops, data)
0462 #define platform_msi_create_device_tree_domain(dev, nvec, write, ops, data) \
0463     __platform_msi_create_device_domain(dev, nvec, true, write, ops, data)
0464
0465 int platform_msi_device_domain_alloc(struct irq_domain *domain, unsigned int virq,
0466                      unsigned int nr_irqs);
0467 void platform_msi_device_domain_free(struct irq_domain *domain, unsigned int virq,
0468                      unsigned int nvec);
0469 void *platform_msi_get_host_data(struct irq_domain *domain);
0470 #endif /* CONFIG_GENERIC_MSI_IRQ_DOMAIN */
0471
0472 #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
0473 struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
0474                          struct msi_domain_info *info,
0475                          struct irq_domain *parent);
0476 u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev);
0477 struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev);
0478 bool pci_dev_has_special_msi_domain(struct pci_dev *pdev);
0479 #else
0480 static inline struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
0481 {
0482     return NULL;
0483 }
0484 #endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */
0485
0486 #endif /* LINUX_MSI_H */