drivers/ntb/msi.c

0001 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
0002
0003 #include <linux/irq.h>
0004 #include <linux/module.h>
0005 #include <linux/ntb.h>
0006 #include <linux/msi.h>
0007 #include <linux/pci.h>
0008
0009 struct ntb_msi {
0010     u64 base_addr;
0011     u64 end_addr;
0012
0013     void (*desc_changed)(void *ctx);
0014
0015     u32 __iomem *peer_mws[];
0016 };
0017
0018 /**
0019  * ntb_msi_init() - Initialize the MSI context
0020  * @ntb:    NTB device context
0021  *
0022  * This function must be called before any other ntb_msi function.
0023  * It initializes the context for MSI operations and maps
0024  * the peer memory windows.
0025  *
0026  * This function reserves the last N outbound memory windows (where N
0027  * is the number of peers).
0028  *
0029  * Return: Zero on success, otherwise a negative error number.
0030  */
0031 int ntb_msi_init(struct ntb_dev *ntb,
0032          void (*desc_changed)(void *ctx))
0033 {
0034     phys_addr_t mw_phys_addr;
0035     resource_size_t mw_size;
0036     int peer_widx;
0037     int peers;
0038     int ret;
0039     int i;
0040
0041     peers = ntb_peer_port_count(ntb);
0042     if (peers <= 0)
0043         return -EINVAL;
0044
0045     ntb->msi = devm_kzalloc(&ntb->dev, struct_size(ntb->msi, peer_mws, peers),
0046                 GFP_KERNEL);
0047     if (!ntb->msi)
0048         return -ENOMEM;
0049
0050     ntb->msi->desc_changed = desc_changed;
0051
0052     for (i = 0; i < peers; i++) {
0053         peer_widx = ntb_peer_mw_count(ntb) - 1 - i;
0054
0055         ret = ntb_peer_mw_get_addr(ntb, peer_widx, &mw_phys_addr,
0056                        &mw_size);
0057         if (ret)
0058             goto unroll;
0059
0060         ntb->msi->peer_mws[i] = devm_ioremap(&ntb->dev, mw_phys_addr,
0061                              mw_size);
0062         if (!ntb->msi->peer_mws[i]) {
0063             ret = -EFAULT;
0064             goto unroll;
0065         }
0066     }
0067
0068     return 0;
0069
0070 unroll:
0071     for (i = 0; i < peers; i++)
0072         if (ntb->msi->peer_mws[i])
0073             devm_iounmap(&ntb->dev, ntb->msi->peer_mws[i]);
0074
0075     devm_kfree(&ntb->dev, ntb->msi);
0076     ntb->msi = NULL;
0077     return ret;
0078 }
0079 EXPORT_SYMBOL(ntb_msi_init);
0080
0081 /**
0082  * ntb_msi_setup_mws() - Initialize the MSI inbound memory windows
0083  * @ntb:    NTB device context
0084  *
0085  * This function sets up the required inbound memory windows. It should be
0086  * called from a work function after a link up event.
0087  *
0088  * Over the entire network, this function will reserves the last N
0089  * inbound memory windows for each peer (where N is the number of peers).
0090  *
0091  * ntb_msi_init() must be called before this function.
0092  *
0093  * Return: Zero on success, otherwise a negative error number.
0094  */
0095 int ntb_msi_setup_mws(struct ntb_dev *ntb)
0096 {
0097     struct msi_desc *desc;
0098     u64 addr;
0099     int peer, peer_widx;
0100     resource_size_t addr_align, size_align, size_max;
0101     resource_size_t mw_size = SZ_32K;
0102     resource_size_t mw_min_size = mw_size;
0103     int i;
0104     int ret;
0105
0106     if (!ntb->msi)
0107         return -EINVAL;
0108
0109     msi_lock_descs(&ntb->pdev->dev);
0110     desc = msi_first_desc(&ntb->pdev->dev, MSI_DESC_ASSOCIATED);
0111     addr = desc->msg.address_lo + ((uint64_t)desc->msg.address_hi << 32);
0112     msi_unlock_descs(&ntb->pdev->dev);
0113
0114     for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) {
0115         peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
0116         if (peer_widx < 0)
0117             return peer_widx;
0118
0119         ret = ntb_mw_get_align(ntb, peer, peer_widx, &addr_align,
0120                        NULL, NULL);
0121         if (ret)
0122             return ret;
0123
0124         addr &= ~(addr_align - 1);
0125     }
0126
0127     for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) {
0128         peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
0129         if (peer_widx < 0) {
0130             ret = peer_widx;
0131             goto error_out;
0132         }
0133
0134         ret = ntb_mw_get_align(ntb, peer, peer_widx, NULL,
0135                        &size_align, &size_max);
0136         if (ret)
0137             goto error_out;
0138
0139         mw_size = round_up(mw_size, size_align);
0140         mw_size = max(mw_size, size_max);
0141         if (mw_size < mw_min_size)
0142             mw_min_size = mw_size;
0143
0144         ret = ntb_mw_set_trans(ntb, peer, peer_widx,
0145                        addr, mw_size);
0146         if (ret)
0147             goto error_out;
0148     }
0149
0150     ntb->msi->base_addr = addr;
0151     ntb->msi->end_addr = addr + mw_min_size;
0152
0153     return 0;
0154
0155 error_out:
0156     for (i = 0; i < peer; i++) {
0157         peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
0158         if (peer_widx < 0)
0159             continue;
0160
0161         ntb_mw_clear_trans(ntb, i, peer_widx);
0162     }
0163
0164     return ret;
0165 }
0166 EXPORT_SYMBOL(ntb_msi_setup_mws);
0167
0168 /**
0169  * ntb_msi_clear_mws() - Clear all inbound memory windows
0170  * @ntb:    NTB device context
0171  *
0172  * This function tears down the resources used by ntb_msi_setup_mws().
0173  */
0174 void ntb_msi_clear_mws(struct ntb_dev *ntb)
0175 {
0176     int peer;
0177     int peer_widx;
0178
0179     for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) {
0180         peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
0181         if (peer_widx < 0)
0182             continue;
0183
0184         ntb_mw_clear_trans(ntb, peer, peer_widx);
0185     }
0186 }
0187 EXPORT_SYMBOL(ntb_msi_clear_mws);
0188
0189 struct ntb_msi_devres {
0190     struct ntb_dev *ntb;
0191     struct msi_desc *entry;
0192     struct ntb_msi_desc *msi_desc;
0193 };
0194
0195 static int ntb_msi_set_desc(struct ntb_dev *ntb, struct msi_desc *entry,
0196                 struct ntb_msi_desc *msi_desc)
0197 {
0198     u64 addr;
0199
0200     addr = entry->msg.address_lo +
0201         ((uint64_t)entry->msg.address_hi << 32);
0202
0203     if (addr < ntb->msi->base_addr || addr >= ntb->msi->end_addr) {
0204         dev_warn_once(&ntb->dev,
0205                   "IRQ %d: MSI Address not within the memory window (%llx, [%llx %llx])\n",
0206                   entry->irq, addr, ntb->msi->base_addr,
0207                   ntb->msi->end_addr);
0208         return -EFAULT;
0209     }
0210
0211     msi_desc->addr_offset = addr - ntb->msi->base_addr;
0212     msi_desc->data = entry->msg.data;
0213
0214     return 0;
0215 }
0216
0217 static void ntb_msi_write_msg(struct msi_desc *entry, void *data)
0218 {
0219     struct ntb_msi_devres *dr = data;
0220
0221     WARN_ON(ntb_msi_set_desc(dr->ntb, entry, dr->msi_desc));
0222
0223     if (dr->ntb->msi->desc_changed)
0224         dr->ntb->msi->desc_changed(dr->ntb->ctx);
0225 }
0226
0227 static void ntbm_msi_callback_release(struct device *dev, void *res)
0228 {
0229     struct ntb_msi_devres *dr = res;
0230
0231     dr->entry->write_msi_msg = NULL;
0232     dr->entry->write_msi_msg_data = NULL;
0233 }
0234
0235 static int ntbm_msi_setup_callback(struct ntb_dev *ntb, struct msi_desc *entry,
0236                    struct ntb_msi_desc *msi_desc)
0237 {
0238     struct ntb_msi_devres *dr;
0239
0240     dr = devres_alloc(ntbm_msi_callback_release,
0241               sizeof(struct ntb_msi_devres), GFP_KERNEL);
0242     if (!dr)
0243         return -ENOMEM;
0244
0245     dr->ntb = ntb;
0246     dr->entry = entry;
0247     dr->msi_desc = msi_desc;
0248
0249     devres_add(&ntb->dev, dr);
0250
0251     dr->entry->write_msi_msg = ntb_msi_write_msg;
0252     dr->entry->write_msi_msg_data = dr;
0253
0254     return 0;
0255 }
0256
0257 /**
0258  * ntbm_msi_request_threaded_irq() - allocate an MSI interrupt
0259  * @ntb:    NTB device context
0260  * @handler:    Function to be called when the IRQ occurs
0261  * @thread_fn:  Function to be called in a threaded interrupt context. NULL
0262  *              for clients which handle everything in @handler
0263  * @name:    An ascii name for the claiming device, dev_name(dev) if NULL
0264  * @dev_id:     A cookie passed back to the handler function
0265  * @msi_desc:   MSI descriptor data which triggers the interrupt
0266  *
0267  * This function assigns an interrupt handler to an unused
0268  * MSI interrupt and returns the descriptor used to trigger
0269  * it. The descriptor can then be sent to a peer to trigger
0270  * the interrupt.
0271  *
0272  * The interrupt resource is managed with devres so it will
0273  * be automatically freed when the NTB device is torn down.
0274  *
0275  * If an IRQ allocated with this function needs to be freed
0276  * separately, ntbm_free_irq() must be used.
0277  *
0278  * Return: IRQ number assigned on success, otherwise a negative error number.
0279  */
0280 int ntbm_msi_request_threaded_irq(struct ntb_dev *ntb, irq_handler_t handler,
0281                   irq_handler_t thread_fn,
0282                   const char *name, void *dev_id,
0283                   struct ntb_msi_desc *msi_desc)
0284 {
0285     struct device *dev = &ntb->pdev->dev;
0286     struct msi_desc *entry;
0287     int ret;
0288
0289     if (!ntb->msi)
0290         return -EINVAL;
0291
0292     msi_lock_descs(dev);
0293     msi_for_each_desc(entry, dev, MSI_DESC_ASSOCIATED) {
0294         if (irq_has_action(entry->irq))
0295             continue;
0296
0297         ret = devm_request_threaded_irq(&ntb->dev, entry->irq, handler,
0298                         thread_fn, 0, name, dev_id);
0299         if (ret)
0300             continue;
0301
0302         if (ntb_msi_set_desc(ntb, entry, msi_desc)) {
0303             devm_free_irq(&ntb->dev, entry->irq, dev_id);
0304             continue;
0305         }
0306
0307         ret = ntbm_msi_setup_callback(ntb, entry, msi_desc);
0308         if (ret) {
0309             devm_free_irq(&ntb->dev, entry->irq, dev_id);
0310             goto unlock;
0311         }
0312
0313         ret = entry->irq;
0314         goto unlock;
0315     }
0316     ret = -ENODEV;
0317
0318 unlock:
0319     msi_unlock_descs(dev);
0320     return ret;
0321 }
0322 EXPORT_SYMBOL(ntbm_msi_request_threaded_irq);
0323
0324 static int ntbm_msi_callback_match(struct device *dev, void *res, void *data)
0325 {
0326     struct ntb_dev *ntb = dev_ntb(dev);
0327     struct ntb_msi_devres *dr = res;
0328
0329     return dr->ntb == ntb && dr->entry == data;
0330 }
0331
0332 /**
0333  * ntbm_msi_free_irq() - free an interrupt
0334  * @ntb:    NTB device context
0335  * @irq:    Interrupt line to free
0336  * @dev_id: Device identity to free
0337  *
0338  * This function should be used to manually free IRQs allocated with
0339  * ntbm_request_[threaded_]irq().
0340  */
0341 void ntbm_msi_free_irq(struct ntb_dev *ntb, unsigned int irq, void *dev_id)
0342 {
0343     struct msi_desc *entry = irq_get_msi_desc(irq);
0344
0345     entry->write_msi_msg = NULL;
0346     entry->write_msi_msg_data = NULL;
0347
0348     WARN_ON(devres_destroy(&ntb->dev, ntbm_msi_callback_release,
0349                    ntbm_msi_callback_match, entry));
0350
0351     devm_free_irq(&ntb->dev, irq, dev_id);
0352 }
0353 EXPORT_SYMBOL(ntbm_msi_free_irq);
0354
0355 /**
0356  * ntb_msi_peer_trigger() - Trigger an interrupt handler on a peer
0357  * @ntb:    NTB device context
0358  * @peer:   Peer index
0359  * @desc:   MSI descriptor data which triggers the interrupt
0360  *
0361  * This function triggers an interrupt on a peer. It requires
0362  * the descriptor structure to have been passed from that peer
0363  * by some other means.
0364  *
0365  * Return: Zero on success, otherwise a negative error number.
0366  */
0367 int ntb_msi_peer_trigger(struct ntb_dev *ntb, int peer,
0368              struct ntb_msi_desc *desc)
0369 {
0370     int idx;
0371
0372     if (!ntb->msi)
0373         return -EINVAL;
0374
0375     idx = desc->addr_offset / sizeof(*ntb->msi->peer_mws[peer]);
0376
0377     iowrite32(desc->data, &ntb->msi->peer_mws[peer][idx]);
0378
0379     return 0;
0380 }
0381 EXPORT_SYMBOL(ntb_msi_peer_trigger);
0382
0383 /**
0384  * ntb_msi_peer_addr() - Get the DMA address to trigger a peer's MSI interrupt
0385  * @ntb:    NTB device context
0386  * @peer:   Peer index
0387  * @desc:   MSI descriptor data which triggers the interrupt
0388  * @msi_addr:   Physical address to trigger the interrupt
0389  *
0390  * This function allows using DMA engines to trigger an interrupt
0391  * (for example, trigger an interrupt to process the data after
0392  * sending it). To trigger the interrupt, write @desc.data to the address
0393  * returned in @msi_addr
0394  *
0395  * Return: Zero on success, otherwise a negative error number.
0396  */
0397 int ntb_msi_peer_addr(struct ntb_dev *ntb, int peer,
0398               struct ntb_msi_desc *desc,
0399               phys_addr_t *msi_addr)
0400 {
0401     int peer_widx = ntb_peer_mw_count(ntb) - 1 - peer;
0402     phys_addr_t mw_phys_addr;
0403     int ret;
0404
0405     ret = ntb_peer_mw_get_addr(ntb, peer_widx, &mw_phys_addr, NULL);
0406     if (ret)
0407         return ret;
0408
0409     if (msi_addr)
0410         *msi_addr = mw_phys_addr + desc->addr_offset;
0411
0412     return 0;
0413 }
0414 EXPORT_SYMBOL(ntb_msi_peer_addr);