hw/hfi1/aspm.c

0001 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
0002 /*
0003  * Copyright(c) 2019 Intel Corporation.
0004  *
0005  */
0006
0007 #include "aspm.h"
0008
0009 /* Time after which the timer interrupt will re-enable ASPM */
0010 #define ASPM_TIMER_MS 1000
0011 /* Time for which interrupts are ignored after a timer has been scheduled */
0012 #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)
0013 /* Two interrupts within this time trigger ASPM disable */
0014 #define ASPM_TRIGGER_MS 1
0015 #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)
0016 #define ASPM_L1_SUPPORTED(reg) \
0017     ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)
0018
0019 uint aspm_mode = ASPM_MODE_DISABLED;
0020 module_param_named(aspm, aspm_mode, uint, 0444);
0021 MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");
0022
0023 static bool aspm_hw_l1_supported(struct hfi1_devdata *dd)
0024 {
0025     struct pci_dev *parent = dd->pcidev->bus->self;
0026     u32 up, dn;
0027
0028     /*
0029      * If the driver does not have access to the upstream component,
0030      * it cannot support ASPM L1 at all.
0031      */
0032     if (!parent)
0033         return false;
0034
0035     pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);
0036     dn = ASPM_L1_SUPPORTED(dn);
0037
0038     pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);
0039     up = ASPM_L1_SUPPORTED(up);
0040
0041     /* ASPM works on A-step but is reported as not supported */
0042     return (!!dn || is_ax(dd)) && !!up;
0043 }
0044
0045 /* Set L1 entrance latency for slower entry to L1 */
0046 static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)
0047 {
0048     u32 l1_ent_lat = 0x4u;
0049     u32 reg32;
0050
0051     pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, &reg32);
0052     reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;
0053     reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;
0054     pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);
0055 }
0056
0057 static void aspm_hw_enable_l1(struct hfi1_devdata *dd)
0058 {
0059     struct pci_dev *parent = dd->pcidev->bus->self;
0060
0061     /*
0062      * If the driver does not have access to the upstream component,
0063      * it cannot support ASPM L1 at all.
0064      */
0065     if (!parent)
0066         return;
0067
0068     /* Enable ASPM L1 first in upstream component and then downstream */
0069     pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
0070                        PCI_EXP_LNKCTL_ASPMC,
0071                        PCI_EXP_LNKCTL_ASPM_L1);
0072     pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
0073                        PCI_EXP_LNKCTL_ASPMC,
0074                        PCI_EXP_LNKCTL_ASPM_L1);
0075 }
0076
0077 void aspm_hw_disable_l1(struct hfi1_devdata *dd)
0078 {
0079     struct pci_dev *parent = dd->pcidev->bus->self;
0080
0081     /* Disable ASPM L1 first in downstream component and then upstream */
0082     pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
0083                        PCI_EXP_LNKCTL_ASPMC, 0x0);
0084     if (parent)
0085         pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
0086                            PCI_EXP_LNKCTL_ASPMC, 0x0);
0087 }
0088
0089 static  void aspm_enable(struct hfi1_devdata *dd)
0090 {
0091     if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||
0092         !dd->aspm_supported)
0093         return;
0094
0095     aspm_hw_enable_l1(dd);
0096     dd->aspm_enabled = true;
0097 }
0098
0099 static  void aspm_disable(struct hfi1_devdata *dd)
0100 {
0101     if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)
0102         return;
0103
0104     aspm_hw_disable_l1(dd);
0105     dd->aspm_enabled = false;
0106 }
0107
0108 static  void aspm_disable_inc(struct hfi1_devdata *dd)
0109 {
0110     unsigned long flags;
0111
0112     spin_lock_irqsave(&dd->aspm_lock, flags);
0113     aspm_disable(dd);
0114     atomic_inc(&dd->aspm_disabled_cnt);
0115     spin_unlock_irqrestore(&dd->aspm_lock, flags);
0116 }
0117
0118 static  void aspm_enable_dec(struct hfi1_devdata *dd)
0119 {
0120     unsigned long flags;
0121
0122     spin_lock_irqsave(&dd->aspm_lock, flags);
0123     if (atomic_dec_and_test(&dd->aspm_disabled_cnt))
0124         aspm_enable(dd);
0125     spin_unlock_irqrestore(&dd->aspm_lock, flags);
0126 }
0127
0128 /* ASPM processing for each receive context interrupt */
0129 void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd)
0130 {
0131     bool restart_timer;
0132     bool close_interrupts;
0133     unsigned long flags;
0134     ktime_t now, prev;
0135
0136     spin_lock_irqsave(&rcd->aspm_lock, flags);
0137     /* PSM contexts are open */
0138     if (!rcd->aspm_intr_enable)
0139         goto unlock;
0140
0141     prev = rcd->aspm_ts_last_intr;
0142     now = ktime_get();
0143     rcd->aspm_ts_last_intr = now;
0144
0145     /* An interrupt pair close together in time */
0146     close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;
0147
0148     /* Don't push out our timer till this much time has elapsed */
0149     restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >
0150                     ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;
0151     restart_timer = restart_timer && close_interrupts;
0152
0153     /* Disable ASPM and schedule timer */
0154     if (rcd->aspm_enabled && close_interrupts) {
0155         aspm_disable_inc(rcd->dd);
0156         rcd->aspm_enabled = false;
0157         restart_timer = true;
0158     }
0159
0160     if (restart_timer) {
0161         mod_timer(&rcd->aspm_timer,
0162               jiffies + msecs_to_jiffies(ASPM_TIMER_MS));
0163         rcd->aspm_ts_timer_sched = now;
0164     }
0165 unlock:
0166     spin_unlock_irqrestore(&rcd->aspm_lock, flags);
0167 }
0168
0169 /* Timer function for re-enabling ASPM in the absence of interrupt activity */
0170 static  void aspm_ctx_timer_function(struct timer_list *t)
0171 {
0172     struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);
0173     unsigned long flags;
0174
0175     spin_lock_irqsave(&rcd->aspm_lock, flags);
0176     aspm_enable_dec(rcd->dd);
0177     rcd->aspm_enabled = true;
0178     spin_unlock_irqrestore(&rcd->aspm_lock, flags);
0179 }
0180
0181 /*
0182  * Disable interrupt processing for verbs contexts when PSM or VNIC contexts
0183  * are open.
0184  */
0185 void aspm_disable_all(struct hfi1_devdata *dd)
0186 {
0187     struct hfi1_ctxtdata *rcd;
0188     unsigned long flags;
0189     u16 i;
0190
0191     for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
0192         rcd = hfi1_rcd_get_by_index(dd, i);
0193         if (rcd) {
0194             del_timer_sync(&rcd->aspm_timer);
0195             spin_lock_irqsave(&rcd->aspm_lock, flags);
0196             rcd->aspm_intr_enable = false;
0197             spin_unlock_irqrestore(&rcd->aspm_lock, flags);
0198             hfi1_rcd_put(rcd);
0199         }
0200     }
0201
0202     aspm_disable(dd);
0203     atomic_set(&dd->aspm_disabled_cnt, 0);
0204 }
0205
0206 /* Re-enable interrupt processing for verbs contexts */
0207 void aspm_enable_all(struct hfi1_devdata *dd)
0208 {
0209     struct hfi1_ctxtdata *rcd;
0210     unsigned long flags;
0211     u16 i;
0212
0213     aspm_enable(dd);
0214
0215     if (aspm_mode != ASPM_MODE_DYNAMIC)
0216         return;
0217
0218     for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
0219         rcd = hfi1_rcd_get_by_index(dd, i);
0220         if (rcd) {
0221             spin_lock_irqsave(&rcd->aspm_lock, flags);
0222             rcd->aspm_intr_enable = true;
0223             rcd->aspm_enabled = true;
0224             spin_unlock_irqrestore(&rcd->aspm_lock, flags);
0225             hfi1_rcd_put(rcd);
0226         }
0227     }
0228 }
0229
0230 static  void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
0231 {
0232     spin_lock_init(&rcd->aspm_lock);
0233     timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);
0234     rcd->aspm_intr_supported = rcd->dd->aspm_supported &&
0235         aspm_mode == ASPM_MODE_DYNAMIC &&
0236         rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;
0237 }
0238
0239 void aspm_init(struct hfi1_devdata *dd)
0240 {
0241     struct hfi1_ctxtdata *rcd;
0242     u16 i;
0243
0244     spin_lock_init(&dd->aspm_lock);
0245     dd->aspm_supported = aspm_hw_l1_supported(dd);
0246
0247     for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
0248         rcd = hfi1_rcd_get_by_index(dd, i);
0249         if (rcd)
0250             aspm_ctx_init(rcd);
0251         hfi1_rcd_put(rcd);
0252     }
0253
0254     /* Start with ASPM disabled */
0255     aspm_hw_set_l1_ent_latency(dd);
0256     dd->aspm_enabled = false;
0257     aspm_hw_disable_l1(dd);
0258
0259     /* Now turn on ASPM if configured */
0260     aspm_enable_all(dd);
0261 }
0262
0263 void aspm_exit(struct hfi1_devdata *dd)
0264 {
0265     aspm_disable_all(dd);
0266
0267     /* Turn on ASPM on exit to conserve power */
0268     aspm_enable(dd);
0269 }
0270