pci/pcie/portdrv_core.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Purpose: PCI Express Port Bus Driver's Core Functions
0004  *
0005  * Copyright (C) 2004 Intel
0006  * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
0007  */
0008
0009 #include <linux/module.h>
0010 #include <linux/pci.h>
0011 #include <linux/kernel.h>
0012 #include <linux/errno.h>
0013 #include <linux/pm.h>
0014 #include <linux/pm_runtime.h>
0015 #include <linux/string.h>
0016 #include <linux/slab.h>
0017 #include <linux/aer.h>
0018
0019 #include "../pci.h"
0020 #include "portdrv.h"
0021
0022 struct portdrv_service_data {
0023     struct pcie_port_service_driver *drv;
0024     struct device *dev;
0025     u32 service;
0026 };
0027
0028 /**
0029  * release_pcie_device - free PCI Express port service device structure
0030  * @dev: Port service device to release
0031  *
0032  * Invoked automatically when device is being removed in response to
0033  * device_unregister(dev).  Release all resources being claimed.
0034  */
0035 static void release_pcie_device(struct device *dev)
0036 {
0037     kfree(to_pcie_device(dev));
0038 }
0039
0040 /*
0041  * Fill in *pme, *aer, *dpc with the relevant Interrupt Message Numbers if
0042  * services are enabled in "mask".  Return the number of MSI/MSI-X vectors
0043  * required to accommodate the largest Message Number.
0044  */
0045 static int pcie_message_numbers(struct pci_dev *dev, int mask,
0046                 u32 *pme, u32 *aer, u32 *dpc)
0047 {
0048     u32 nvec = 0, pos;
0049     u16 reg16;
0050
0051     /*
0052      * The Interrupt Message Number indicates which vector is used, i.e.,
0053      * the MSI-X table entry or the MSI offset between the base Message
0054      * Data and the generated interrupt message.  See PCIe r3.1, sec
0055      * 7.8.2, 7.10.10, 7.31.2.
0056      */
0057
0058     if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP |
0059             PCIE_PORT_SERVICE_BWNOTIF)) {
0060         pcie_capability_read_word(dev, PCI_EXP_FLAGS, &reg16);
0061         *pme = (reg16 & PCI_EXP_FLAGS_IRQ) >> 9;
0062         nvec = *pme + 1;
0063     }
0064
0065 #ifdef CONFIG_PCIEAER
0066     if (mask & PCIE_PORT_SERVICE_AER) {
0067         u32 reg32;
0068
0069         pos = dev->aer_cap;
0070         if (pos) {
0071             pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS,
0072                           &reg32);
0073             *aer = (reg32 & PCI_ERR_ROOT_AER_IRQ) >> 27;
0074             nvec = max(nvec, *aer + 1);
0075         }
0076     }
0077 #endif
0078
0079     if (mask & PCIE_PORT_SERVICE_DPC) {
0080         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC);
0081         if (pos) {
0082             pci_read_config_word(dev, pos + PCI_EXP_DPC_CAP,
0083                          &reg16);
0084             *dpc = reg16 & PCI_EXP_DPC_IRQ;
0085             nvec = max(nvec, *dpc + 1);
0086         }
0087     }
0088
0089     return nvec;
0090 }
0091
0092 /**
0093  * pcie_port_enable_irq_vec - try to set up MSI-X or MSI as interrupt mode
0094  * for given port
0095  * @dev: PCI Express port to handle
0096  * @irqs: Array of interrupt vectors to populate
0097  * @mask: Bitmask of port capabilities returned by get_port_device_capability()
0098  *
0099  * Return value: 0 on success, error code on failure
0100  */
0101 static int pcie_port_enable_irq_vec(struct pci_dev *dev, int *irqs, int mask)
0102 {
0103     int nr_entries, nvec, pcie_irq;
0104     u32 pme = 0, aer = 0, dpc = 0;
0105
0106     /* Allocate the maximum possible number of MSI/MSI-X vectors */
0107     nr_entries = pci_alloc_irq_vectors(dev, 1, PCIE_PORT_MAX_MSI_ENTRIES,
0108             PCI_IRQ_MSIX | PCI_IRQ_MSI);
0109     if (nr_entries < 0)
0110         return nr_entries;
0111
0112     /* See how many and which Interrupt Message Numbers we actually use */
0113     nvec = pcie_message_numbers(dev, mask, &pme, &aer, &dpc);
0114     if (nvec > nr_entries) {
0115         pci_free_irq_vectors(dev);
0116         return -EIO;
0117     }
0118
0119     /*
0120      * If we allocated more than we need, free them and reallocate fewer.
0121      *
0122      * Reallocating may change the specific vectors we get, so
0123      * pci_irq_vector() must be done *after* the reallocation.
0124      *
0125      * If we're using MSI, hardware is *allowed* to change the Interrupt
0126      * Message Numbers when we free and reallocate the vectors, but we
0127      * assume it won't because we allocate enough vectors for the
0128      * biggest Message Number we found.
0129      */
0130     if (nvec != nr_entries) {
0131         pci_free_irq_vectors(dev);
0132
0133         nr_entries = pci_alloc_irq_vectors(dev, nvec, nvec,
0134                 PCI_IRQ_MSIX | PCI_IRQ_MSI);
0135         if (nr_entries < 0)
0136             return nr_entries;
0137     }
0138
0139     /* PME, hotplug and bandwidth notification share an MSI/MSI-X vector */
0140     if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP |
0141             PCIE_PORT_SERVICE_BWNOTIF)) {
0142         pcie_irq = pci_irq_vector(dev, pme);
0143         irqs[PCIE_PORT_SERVICE_PME_SHIFT] = pcie_irq;
0144         irqs[PCIE_PORT_SERVICE_HP_SHIFT] = pcie_irq;
0145         irqs[PCIE_PORT_SERVICE_BWNOTIF_SHIFT] = pcie_irq;
0146     }
0147
0148     if (mask & PCIE_PORT_SERVICE_AER)
0149         irqs[PCIE_PORT_SERVICE_AER_SHIFT] = pci_irq_vector(dev, aer);
0150
0151     if (mask & PCIE_PORT_SERVICE_DPC)
0152         irqs[PCIE_PORT_SERVICE_DPC_SHIFT] = pci_irq_vector(dev, dpc);
0153
0154     return 0;
0155 }
0156
0157 /**
0158  * pcie_init_service_irqs - initialize irqs for PCI Express port services
0159  * @dev: PCI Express port to handle
0160  * @irqs: Array of irqs to populate
0161  * @mask: Bitmask of port capabilities returned by get_port_device_capability()
0162  *
0163  * Return value: Interrupt mode associated with the port
0164  */
0165 static int pcie_init_service_irqs(struct pci_dev *dev, int *irqs, int mask)
0166 {
0167     int ret, i;
0168
0169     for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
0170         irqs[i] = -1;
0171
0172     /*
0173      * If we support PME but can't use MSI/MSI-X for it, we have to
0174      * fall back to INTx or other interrupts, e.g., a system shared
0175      * interrupt.
0176      */
0177     if ((mask & PCIE_PORT_SERVICE_PME) && pcie_pme_no_msi())
0178         goto legacy_irq;
0179
0180     /* Try to use MSI-X or MSI if supported */
0181     if (pcie_port_enable_irq_vec(dev, irqs, mask) == 0)
0182         return 0;
0183
0184 legacy_irq:
0185     /* fall back to legacy IRQ */
0186     ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_LEGACY);
0187     if (ret < 0)
0188         return -ENODEV;
0189
0190     for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++)
0191         irqs[i] = pci_irq_vector(dev, 0);
0192
0193     return 0;
0194 }
0195
0196 /**
0197  * get_port_device_capability - discover capabilities of a PCI Express port
0198  * @dev: PCI Express port to examine
0199  *
0200  * The capabilities are read from the port's PCI Express configuration registers
0201  * as described in PCI Express Base Specification 1.0a sections 7.8.2, 7.8.9 and
0202  * 7.9 - 7.11.
0203  *
0204  * Return value: Bitmask of discovered port capabilities
0205  */
0206 static int get_port_device_capability(struct pci_dev *dev)
0207 {
0208     struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
0209     int services = 0;
0210
0211     if (dev->is_hotplug_bridge &&
0212         (pcie_ports_native || host->native_pcie_hotplug)) {
0213         services |= PCIE_PORT_SERVICE_HP;
0214
0215         /*
0216          * Disable hot-plug interrupts in case they have been enabled
0217          * by the BIOS and the hot-plug service driver is not loaded.
0218          */
0219         pcie_capability_clear_word(dev, PCI_EXP_SLTCTL,
0220               PCI_EXP_SLTCTL_CCIE | PCI_EXP_SLTCTL_HPIE);
0221     }
0222
0223 #ifdef CONFIG_PCIEAER
0224     if (dev->aer_cap && pci_aer_available() &&
0225         (pcie_ports_native || host->native_aer))
0226         services |= PCIE_PORT_SERVICE_AER;
0227 #endif
0228
0229     /* Root Ports and Root Complex Event Collectors may generate PMEs */
0230     if ((pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
0231          pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC) &&
0232         (pcie_ports_native || host->native_pme)) {
0233         services |= PCIE_PORT_SERVICE_PME;
0234
0235         /*
0236          * Disable PME interrupt on this port in case it's been enabled
0237          * by the BIOS (the PME service driver will enable it when
0238          * necessary).
0239          */
0240         pcie_pme_interrupt_enable(dev, false);
0241     }
0242
0243     /*
0244      * With dpc-native, allow Linux to use DPC even if it doesn't have
0245      * permission to use AER.
0246      */
0247     if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC) &&
0248         pci_aer_available() &&
0249         (pcie_ports_dpc_native || (services & PCIE_PORT_SERVICE_AER)))
0250         services |= PCIE_PORT_SERVICE_DPC;
0251
0252     if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM ||
0253         pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) {
0254         u32 linkcap;
0255
0256         pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &linkcap);
0257         if (linkcap & PCI_EXP_LNKCAP_LBNC)
0258             services |= PCIE_PORT_SERVICE_BWNOTIF;
0259     }
0260
0261     return services;
0262 }
0263
0264 /**
0265  * pcie_device_init - allocate and initialize PCI Express port service device
0266  * @pdev: PCI Express port to associate the service device with
0267  * @service: Type of service to associate with the service device
0268  * @irq: Interrupt vector to associate with the service device
0269  */
0270 static int pcie_device_init(struct pci_dev *pdev, int service, int irq)
0271 {
0272     int retval;
0273     struct pcie_device *pcie;
0274     struct device *device;
0275
0276     pcie = kzalloc(sizeof(*pcie), GFP_KERNEL);
0277     if (!pcie)
0278         return -ENOMEM;
0279     pcie->port = pdev;
0280     pcie->irq = irq;
0281     pcie->service = service;
0282
0283     /* Initialize generic device interface */
0284     device = &pcie->device;
0285     device->bus = &pcie_port_bus_type;
0286     device->release = release_pcie_device;  /* callback to free pcie dev */
0287     dev_set_name(device, "%s:pcie%03x",
0288              pci_name(pdev),
0289              get_descriptor_id(pci_pcie_type(pdev), service));
0290     device->parent = &pdev->dev;
0291     device_enable_async_suspend(device);
0292
0293     retval = device_register(device);
0294     if (retval) {
0295         put_device(device);
0296         return retval;
0297     }
0298
0299     pm_runtime_no_callbacks(device);
0300
0301     return 0;
0302 }
0303
0304 /**
0305  * pcie_port_device_register - register PCI Express port
0306  * @dev: PCI Express port to register
0307  *
0308  * Allocate the port extension structure and register services associated with
0309  * the port.
0310  */
0311 int pcie_port_device_register(struct pci_dev *dev)
0312 {
0313     int status, capabilities, i, nr_service;
0314     int irqs[PCIE_PORT_DEVICE_MAXSERVICES];
0315
0316     /* Enable PCI Express port device */
0317     status = pci_enable_device(dev);
0318     if (status)
0319         return status;
0320
0321     /* Get and check PCI Express port services */
0322     capabilities = get_port_device_capability(dev);
0323     if (!capabilities)
0324         return 0;
0325
0326     pci_set_master(dev);
0327     /*
0328      * Initialize service irqs. Don't use service devices that
0329      * require interrupts if there is no way to generate them.
0330      * However, some drivers may have a polling mode (e.g. pciehp_poll_mode)
0331      * that can be used in the absence of irqs.  Allow them to determine
0332      * if that is to be used.
0333      */
0334     status = pcie_init_service_irqs(dev, irqs, capabilities);
0335     if (status) {
0336         capabilities &= PCIE_PORT_SERVICE_HP;
0337         if (!capabilities)
0338             goto error_disable;
0339     }
0340
0341     /* Allocate child services if any */
0342     status = -ENODEV;
0343     nr_service = 0;
0344     for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) {
0345         int service = 1 << i;
0346         if (!(capabilities & service))
0347             continue;
0348         if (!pcie_device_init(dev, service, irqs[i]))
0349             nr_service++;
0350     }
0351     if (!nr_service)
0352         goto error_cleanup_irqs;
0353
0354     return 0;
0355
0356 error_cleanup_irqs:
0357     pci_free_irq_vectors(dev);
0358 error_disable:
0359     pci_disable_device(dev);
0360     return status;
0361 }
0362
0363 typedef int (*pcie_callback_t)(struct pcie_device *);
0364
0365 int pcie_port_device_iter(struct device *dev, void *data)
0366 {
0367     struct pcie_port_service_driver *service_driver;
0368     size_t offset = *(size_t *)data;
0369     pcie_callback_t cb;
0370
0371     if ((dev->bus == &pcie_port_bus_type) && dev->driver) {
0372         service_driver = to_service_driver(dev->driver);
0373         cb = *(pcie_callback_t *)((void *)service_driver + offset);
0374         if (cb)
0375             return cb(to_pcie_device(dev));
0376     }
0377     return 0;
0378 }
0379
0380 #ifdef CONFIG_PM
0381 /**
0382  * pcie_port_device_suspend - suspend port services associated with a PCIe port
0383  * @dev: PCI Express port to handle
0384  */
0385 int pcie_port_device_suspend(struct device *dev)
0386 {
0387     size_t off = offsetof(struct pcie_port_service_driver, suspend);
0388     return device_for_each_child(dev, &off, pcie_port_device_iter);
0389 }
0390
0391 int pcie_port_device_resume_noirq(struct device *dev)
0392 {
0393     size_t off = offsetof(struct pcie_port_service_driver, resume_noirq);
0394     return device_for_each_child(dev, &off, pcie_port_device_iter);
0395 }
0396
0397 /**
0398  * pcie_port_device_resume - resume port services associated with a PCIe port
0399  * @dev: PCI Express port to handle
0400  */
0401 int pcie_port_device_resume(struct device *dev)
0402 {
0403     size_t off = offsetof(struct pcie_port_service_driver, resume);
0404     return device_for_each_child(dev, &off, pcie_port_device_iter);
0405 }
0406
0407 /**
0408  * pcie_port_device_runtime_suspend - runtime suspend port services
0409  * @dev: PCI Express port to handle
0410  */
0411 int pcie_port_device_runtime_suspend(struct device *dev)
0412 {
0413     size_t off = offsetof(struct pcie_port_service_driver, runtime_suspend);
0414     return device_for_each_child(dev, &off, pcie_port_device_iter);
0415 }
0416
0417 /**
0418  * pcie_port_device_runtime_resume - runtime resume port services
0419  * @dev: PCI Express port to handle
0420  */
0421 int pcie_port_device_runtime_resume(struct device *dev)
0422 {
0423     size_t off = offsetof(struct pcie_port_service_driver, runtime_resume);
0424     return device_for_each_child(dev, &off, pcie_port_device_iter);
0425 }
0426 #endif /* PM */
0427
0428 static int remove_iter(struct device *dev, void *data)
0429 {
0430     if (dev->bus == &pcie_port_bus_type)
0431         device_unregister(dev);
0432     return 0;
0433 }
0434
0435 static int find_service_iter(struct device *device, void *data)
0436 {
0437     struct pcie_port_service_driver *service_driver;
0438     struct portdrv_service_data *pdrvs;
0439     u32 service;
0440
0441     pdrvs = (struct portdrv_service_data *) data;
0442     service = pdrvs->service;
0443
0444     if (device->bus == &pcie_port_bus_type && device->driver) {
0445         service_driver = to_service_driver(device->driver);
0446         if (service_driver->service == service) {
0447             pdrvs->drv = service_driver;
0448             pdrvs->dev = device;
0449             return 1;
0450         }
0451     }
0452
0453     return 0;
0454 }
0455
0456 /**
0457  * pcie_port_find_device - find the struct device
0458  * @dev: PCI Express port the service is associated with
0459  * @service: For the service to find
0460  *
0461  * Find the struct device associated with given service on a pci_dev
0462  */
0463 struct device *pcie_port_find_device(struct pci_dev *dev,
0464                       u32 service)
0465 {
0466     struct device *device;
0467     struct portdrv_service_data pdrvs;
0468
0469     pdrvs.dev = NULL;
0470     pdrvs.service = service;
0471     device_for_each_child(&dev->dev, &pdrvs, find_service_iter);
0472
0473     device = pdrvs.dev;
0474     return device;
0475 }
0476 EXPORT_SYMBOL_GPL(pcie_port_find_device);
0477
0478 /**
0479  * pcie_port_device_remove - unregister PCI Express port service devices
0480  * @dev: PCI Express port the service devices to unregister are associated with
0481  *
0482  * Remove PCI Express port service devices associated with given port and
0483  * disable MSI-X or MSI for the port.
0484  */
0485 void pcie_port_device_remove(struct pci_dev *dev)
0486 {
0487     device_for_each_child(&dev->dev, NULL, remove_iter);
0488     pci_free_irq_vectors(dev);
0489     pci_disable_device(dev);
0490 }
0491
0492 /**
0493  * pcie_port_probe_service - probe driver for given PCI Express port service
0494  * @dev: PCI Express port service device to probe against
0495  *
0496  * If PCI Express port service driver is registered with
0497  * pcie_port_service_register(), this function will be called by the driver core
0498  * whenever match is found between the driver and a port service device.
0499  */
0500 static int pcie_port_probe_service(struct device *dev)
0501 {
0502     struct pcie_device *pciedev;
0503     struct pcie_port_service_driver *driver;
0504     int status;
0505
0506     if (!dev || !dev->driver)
0507         return -ENODEV;
0508
0509     driver = to_service_driver(dev->driver);
0510     if (!driver || !driver->probe)
0511         return -ENODEV;
0512
0513     pciedev = to_pcie_device(dev);
0514     status = driver->probe(pciedev);
0515     if (status)
0516         return status;
0517
0518     get_device(dev);
0519     return 0;
0520 }
0521
0522 /**
0523  * pcie_port_remove_service - detach driver from given PCI Express port service
0524  * @dev: PCI Express port service device to handle
0525  *
0526  * If PCI Express port service driver is registered with
0527  * pcie_port_service_register(), this function will be called by the driver core
0528  * when device_unregister() is called for the port service device associated
0529  * with the driver.
0530  */
0531 static int pcie_port_remove_service(struct device *dev)
0532 {
0533     struct pcie_device *pciedev;
0534     struct pcie_port_service_driver *driver;
0535
0536     if (!dev || !dev->driver)
0537         return 0;
0538
0539     pciedev = to_pcie_device(dev);
0540     driver = to_service_driver(dev->driver);
0541     if (driver && driver->remove) {
0542         driver->remove(pciedev);
0543         put_device(dev);
0544     }
0545     return 0;
0546 }
0547
0548 /**
0549  * pcie_port_shutdown_service - shut down given PCI Express port service
0550  * @dev: PCI Express port service device to handle
0551  *
0552  * If PCI Express port service driver is registered with
0553  * pcie_port_service_register(), this function will be called by the driver core
0554  * when device_shutdown() is called for the port service device associated
0555  * with the driver.
0556  */
0557 static void pcie_port_shutdown_service(struct device *dev) {}
0558
0559 /**
0560  * pcie_port_service_register - register PCI Express port service driver
0561  * @new: PCI Express port service driver to register
0562  */
0563 int pcie_port_service_register(struct pcie_port_service_driver *new)
0564 {
0565     if (pcie_ports_disabled)
0566         return -ENODEV;
0567
0568     new->driver.name = new->name;
0569     new->driver.bus = &pcie_port_bus_type;
0570     new->driver.probe = pcie_port_probe_service;
0571     new->driver.remove = pcie_port_remove_service;
0572     new->driver.shutdown = pcie_port_shutdown_service;
0573
0574     return driver_register(&new->driver);
0575 }
0576 EXPORT_SYMBOL(pcie_port_service_register);
0577
0578 /**
0579  * pcie_port_service_unregister - unregister PCI Express port service driver
0580  * @drv: PCI Express port service driver to unregister
0581  */
0582 void pcie_port_service_unregister(struct pcie_port_service_driver *drv)
0583 {
0584     driver_unregister(&drv->driver);
0585 }
0586 EXPORT_SYMBOL(pcie_port_service_unregister);