drivers/pci/vc.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * PCI Virtual Channel support
0004  *
0005  * Copyright (C) 2013 Red Hat, Inc.  All rights reserved.
0006  *     Author: Alex Williamson <alex.williamson@redhat.com>
0007  */
0008
0009 #include <linux/device.h>
0010 #include <linux/kernel.h>
0011 #include <linux/module.h>
0012 #include <linux/pci.h>
0013 #include <linux/pci_regs.h>
0014 #include <linux/types.h>
0015
0016 #include "pci.h"
0017
0018 /**
0019  * pci_vc_save_restore_dwords - Save or restore a series of dwords
0020  * @dev: device
0021  * @pos: starting config space position
0022  * @buf: buffer to save to or restore from
0023  * @dwords: number of dwords to save/restore
0024  * @save: whether to save or restore
0025  */
0026 static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos,
0027                        u32 *buf, int dwords, bool save)
0028 {
0029     int i;
0030
0031     for (i = 0; i < dwords; i++, buf++) {
0032         if (save)
0033             pci_read_config_dword(dev, pos + (i * 4), buf);
0034         else
0035             pci_write_config_dword(dev, pos + (i * 4), *buf);
0036     }
0037 }
0038
0039 /**
0040  * pci_vc_load_arb_table - load and wait for VC arbitration table
0041  * @dev: device
0042  * @pos: starting position of VC capability (VC/VC9/MFVC)
0043  *
0044  * Set Load VC Arbitration Table bit requesting hardware to apply the VC
0045  * Arbitration Table (previously loaded).  When the VC Arbitration Table
0046  * Status clears, hardware has latched the table into VC arbitration logic.
0047  */
0048 static void pci_vc_load_arb_table(struct pci_dev *dev, int pos)
0049 {
0050     u16 ctrl;
0051
0052     pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl);
0053     pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL,
0054                   ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE);
0055     if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS,
0056                  PCI_VC_PORT_STATUS_TABLE))
0057         return;
0058
0059     pci_err(dev, "VC arbitration table failed to load\n");
0060 }
0061
0062 /**
0063  * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table
0064  * @dev: device
0065  * @pos: starting position of VC capability (VC/VC9/MFVC)
0066  * @res: VC resource number, ie. VCn (0-7)
0067  *
0068  * Set Load Port Arbitration Table bit requesting hardware to apply the Port
0069  * Arbitration Table (previously loaded).  When the Port Arbitration Table
0070  * Status clears, hardware has latched the table into port arbitration logic.
0071  */
0072 static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res)
0073 {
0074     int ctrl_pos, status_pos;
0075     u32 ctrl;
0076
0077     ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
0078     status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
0079
0080     pci_read_config_dword(dev, ctrl_pos, &ctrl);
0081     pci_write_config_dword(dev, ctrl_pos,
0082                    ctrl | PCI_VC_RES_CTRL_LOAD_TABLE);
0083
0084     if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE))
0085         return;
0086
0087     pci_err(dev, "VC%d port arbitration table failed to load\n", res);
0088 }
0089
0090 /**
0091  * pci_vc_enable - Enable virtual channel
0092  * @dev: device
0093  * @pos: starting position of VC capability (VC/VC9/MFVC)
0094  * @res: VC res number, ie. VCn (0-7)
0095  *
0096  * A VC is enabled by setting the enable bit in matching resource control
0097  * registers on both sides of a link.  We therefore need to find the opposite
0098  * end of the link.  To keep this simple we enable from the downstream device.
0099  * RC devices do not have an upstream device, nor does it seem that VC9 do
0100  * (spec is unclear).  Once we find the upstream device, match the VC ID to
0101  * get the correct resource, disable and enable on both ends.
0102  */
0103 static void pci_vc_enable(struct pci_dev *dev, int pos, int res)
0104 {
0105     int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2;
0106     u32 ctrl, header, cap1, ctrl2;
0107     struct pci_dev *link = NULL;
0108
0109     /* Enable VCs from the downstream device */
0110     if (!pci_is_pcie(dev) || !pcie_downstream_port(dev))
0111         return;
0112
0113     ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
0114     status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
0115
0116     pci_read_config_dword(dev, ctrl_pos, &ctrl);
0117     id = ctrl & PCI_VC_RES_CTRL_ID;
0118
0119     pci_read_config_dword(dev, pos, &header);
0120
0121     /* If there is no opposite end of the link, skip to enable */
0122     if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 ||
0123         pci_is_root_bus(dev->bus))
0124         goto enable;
0125
0126     pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC);
0127     if (!pos2)
0128         goto enable;
0129
0130     pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1);
0131     evcc = cap1 & PCI_VC_CAP1_EVCC;
0132
0133     /* VC0 is hardwired enabled, so we can start with 1 */
0134     for (i = 1; i < evcc + 1; i++) {
0135         ctrl_pos2 = pos2 + PCI_VC_RES_CTRL +
0136                 (i * PCI_CAP_VC_PER_VC_SIZEOF);
0137         status_pos2 = pos2 + PCI_VC_RES_STATUS +
0138                 (i * PCI_CAP_VC_PER_VC_SIZEOF);
0139         pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2);
0140         if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) {
0141             link = dev->bus->self;
0142             break;
0143         }
0144     }
0145
0146     if (!link)
0147         goto enable;
0148
0149     /* Disable if enabled */
0150     if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) {
0151         ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE;
0152         pci_write_config_dword(link, ctrl_pos2, ctrl2);
0153     }
0154
0155     /* Enable on both ends */
0156     ctrl2 |= PCI_VC_RES_CTRL_ENABLE;
0157     pci_write_config_dword(link, ctrl_pos2, ctrl2);
0158 enable:
0159     ctrl |= PCI_VC_RES_CTRL_ENABLE;
0160     pci_write_config_dword(dev, ctrl_pos, ctrl);
0161
0162     if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO))
0163         pci_err(dev, "VC%d negotiation stuck pending\n", id);
0164
0165     if (link && !pci_wait_for_pending(link, status_pos2,
0166                       PCI_VC_RES_STATUS_NEGO))
0167         pci_err(link, "VC%d negotiation stuck pending\n", id);
0168 }
0169
0170 /**
0171  * pci_vc_do_save_buffer - Size, save, or restore VC state
0172  * @dev: device
0173  * @pos: starting position of VC capability (VC/VC9/MFVC)
0174  * @save_state: buffer for save/restore
0175  * @save: if provided a buffer, this indicates what to do with it
0176  *
0177  * Walking Virtual Channel config space to size, save, or restore it
0178  * is complicated, so we do it all from one function to reduce code and
0179  * guarantee ordering matches in the buffer.  When called with NULL
0180  * @save_state, return the size of the necessary save buffer.  When called
0181  * with a non-NULL @save_state, @save determines whether we save to the
0182  * buffer or restore from it.
0183  */
0184 static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
0185                  struct pci_cap_saved_state *save_state,
0186                  bool save)
0187 {
0188     u32 cap1;
0189     char evcc, lpevcc, parb_size;
0190     int i, len = 0;
0191     u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL;
0192
0193     /* Sanity check buffer size for save/restore */
0194     if (buf && save_state->cap.size !=
0195         pci_vc_do_save_buffer(dev, pos, NULL, save)) {
0196         pci_err(dev, "VC save buffer size does not match @0x%x\n", pos);
0197         return -ENOMEM;
0198     }
0199
0200     pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1);
0201     /* Extended VC Count (not counting VC0) */
0202     evcc = cap1 & PCI_VC_CAP1_EVCC;
0203     /* Low Priority Extended VC Count (not counting VC0) */
0204     lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4;
0205     /* Port Arbitration Table Entry Size (bits) */
0206     parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10);
0207
0208     /*
0209      * Port VC Control Register contains VC Arbitration Select, which
0210      * cannot be modified when more than one LPVC is in operation.  We
0211      * therefore save/restore it first, as only VC0 should be enabled
0212      * after device reset.
0213      */
0214     if (buf) {
0215         if (save)
0216             pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL,
0217                          (u16 *)buf);
0218         else
0219             pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL,
0220                           *(u16 *)buf);
0221         buf += 4;
0222     }
0223     len += 4;
0224
0225     /*
0226      * If we have any Low Priority VCs and a VC Arbitration Table Offset
0227      * in Port VC Capability Register 2 then save/restore it next.
0228      */
0229     if (lpevcc) {
0230         u32 cap2;
0231         int vcarb_offset;
0232
0233         pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2);
0234         vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16;
0235
0236         if (vcarb_offset) {
0237             int size, vcarb_phases = 0;
0238
0239             if (cap2 & PCI_VC_CAP2_128_PHASE)
0240                 vcarb_phases = 128;
0241             else if (cap2 & PCI_VC_CAP2_64_PHASE)
0242                 vcarb_phases = 64;
0243             else if (cap2 & PCI_VC_CAP2_32_PHASE)
0244                 vcarb_phases = 32;
0245
0246             /* Fixed 4 bits per phase per lpevcc (plus VC0) */
0247             size = ((lpevcc + 1) * vcarb_phases * 4) / 8;
0248
0249             if (size && buf) {
0250                 pci_vc_save_restore_dwords(dev,
0251                                pos + vcarb_offset,
0252                                (u32 *)buf,
0253                                size / 4, save);
0254                 /*
0255                  * On restore, we need to signal hardware to
0256                  * re-load the VC Arbitration Table.
0257                  */
0258                 if (!save)
0259                     pci_vc_load_arb_table(dev, pos);
0260
0261                 buf += size;
0262             }
0263             len += size;
0264         }
0265     }
0266
0267     /*
0268      * In addition to each VC Resource Control Register, we may have a
0269      * Port Arbitration Table attached to each VC.  The Port Arbitration
0270      * Table Offset in each VC Resource Capability Register tells us if
0271      * it exists.  The entry size is global from the Port VC Capability
0272      * Register1 above.  The number of phases is determined per VC.
0273      */
0274     for (i = 0; i < evcc + 1; i++) {
0275         u32 cap;
0276         int parb_offset;
0277
0278         pci_read_config_dword(dev, pos + PCI_VC_RES_CAP +
0279                       (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap);
0280         parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16;
0281         if (parb_offset) {
0282             int size, parb_phases = 0;
0283
0284             if (cap & PCI_VC_RES_CAP_256_PHASE)
0285                 parb_phases = 256;
0286             else if (cap & (PCI_VC_RES_CAP_128_PHASE |
0287                     PCI_VC_RES_CAP_128_PHASE_TB))
0288                 parb_phases = 128;
0289             else if (cap & PCI_VC_RES_CAP_64_PHASE)
0290                 parb_phases = 64;
0291             else if (cap & PCI_VC_RES_CAP_32_PHASE)
0292                 parb_phases = 32;
0293
0294             size = (parb_size * parb_phases) / 8;
0295
0296             if (size && buf) {
0297                 pci_vc_save_restore_dwords(dev,
0298                                pos + parb_offset,
0299                                (u32 *)buf,
0300                                size / 4, save);
0301                 buf += size;
0302             }
0303             len += size;
0304         }
0305
0306         /* VC Resource Control Register */
0307         if (buf) {
0308             int ctrl_pos = pos + PCI_VC_RES_CTRL +
0309                         (i * PCI_CAP_VC_PER_VC_SIZEOF);
0310             if (save)
0311                 pci_read_config_dword(dev, ctrl_pos,
0312                               (u32 *)buf);
0313             else {
0314                 u32 tmp, ctrl = *(u32 *)buf;
0315                 /*
0316                  * For an FLR case, the VC config may remain.
0317                  * Preserve enable bit, restore the rest.
0318                  */
0319                 pci_read_config_dword(dev, ctrl_pos, &tmp);
0320                 tmp &= PCI_VC_RES_CTRL_ENABLE;
0321                 tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE;
0322                 pci_write_config_dword(dev, ctrl_pos, tmp);
0323                 /* Load port arbitration table if used */
0324                 if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT)
0325                     pci_vc_load_port_arb_table(dev, pos, i);
0326                 /* Re-enable if needed */
0327                 if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE)
0328                     pci_vc_enable(dev, pos, i);
0329             }
0330             buf += 4;
0331         }
0332         len += 4;
0333     }
0334
0335     return buf ? 0 : len;
0336 }
0337
0338 static struct {
0339     u16 id;
0340     const char *name;
0341 } vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" },
0342         { PCI_EXT_CAP_ID_VC, "VC" },
0343         { PCI_EXT_CAP_ID_VC9, "VC9" } };
0344
0345 /**
0346  * pci_save_vc_state - Save VC state to pre-allocate save buffer
0347  * @dev: device
0348  *
0349  * For each type of VC capability, VC/VC9/MFVC, find the capability and
0350  * save it to the pre-allocated save buffer.
0351  */
0352 int pci_save_vc_state(struct pci_dev *dev)
0353 {
0354     int i;
0355
0356     for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
0357         int pos, ret;
0358         struct pci_cap_saved_state *save_state;
0359
0360         pos = pci_find_ext_capability(dev, vc_caps[i].id);
0361         if (!pos)
0362             continue;
0363
0364         save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id);
0365         if (!save_state) {
0366             pci_err(dev, "%s buffer not found in %s\n",
0367                 vc_caps[i].name, __func__);
0368             return -ENOMEM;
0369         }
0370
0371         ret = pci_vc_do_save_buffer(dev, pos, save_state, true);
0372         if (ret) {
0373             pci_err(dev, "%s save unsuccessful %s\n",
0374                 vc_caps[i].name, __func__);
0375             return ret;
0376         }
0377     }
0378
0379     return 0;
0380 }
0381
0382 /**
0383  * pci_restore_vc_state - Restore VC state from save buffer
0384  * @dev: device
0385  *
0386  * For each type of VC capability, VC/VC9/MFVC, find the capability and
0387  * restore it from the previously saved buffer.
0388  */
0389 void pci_restore_vc_state(struct pci_dev *dev)
0390 {
0391     int i;
0392
0393     for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
0394         int pos;
0395         struct pci_cap_saved_state *save_state;
0396
0397         pos = pci_find_ext_capability(dev, vc_caps[i].id);
0398         save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id);
0399         if (!save_state || !pos)
0400             continue;
0401
0402         pci_vc_do_save_buffer(dev, pos, save_state, false);
0403     }
0404 }
0405
0406 /**
0407  * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps
0408  * @dev: device
0409  *
0410  * For each type of VC capability, VC/VC9/MFVC, find the capability, size
0411  * it, and allocate a buffer for save/restore.
0412  */
0413 void pci_allocate_vc_save_buffers(struct pci_dev *dev)
0414 {
0415     int i;
0416
0417     for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
0418         int len, pos = pci_find_ext_capability(dev, vc_caps[i].id);
0419
0420         if (!pos)
0421             continue;
0422
0423         len = pci_vc_do_save_buffer(dev, pos, NULL, false);
0424         if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len))
0425             pci_err(dev, "unable to preallocate %s save buffer\n",
0426                 vc_caps[i].name);
0427     }
0428 }