drivers/pci/ats.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * PCI Express I/O Virtualization (IOV) support
0004  *   Address Translation Service 1.0
0005  *   Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
0006  *   PASID support added by Joerg Roedel <joerg.roedel@amd.com>
0007  *
0008  * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
0009  * Copyright (C) 2011 Advanced Micro Devices,
0010  */
0011
0012 #include <linux/export.h>
0013 #include <linux/pci-ats.h>
0014 #include <linux/pci.h>
0015 #include <linux/slab.h>
0016
0017 #include "pci.h"
0018
0019 void pci_ats_init(struct pci_dev *dev)
0020 {
0021     int pos;
0022
0023     if (pci_ats_disabled())
0024         return;
0025
0026     pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
0027     if (!pos)
0028         return;
0029
0030     dev->ats_cap = pos;
0031 }
0032
0033 /**
0034  * pci_ats_supported - check if the device can use ATS
0035  * @dev: the PCI device
0036  *
0037  * Returns true if the device supports ATS and is allowed to use it, false
0038  * otherwise.
0039  */
0040 bool pci_ats_supported(struct pci_dev *dev)
0041 {
0042     if (!dev->ats_cap)
0043         return false;
0044
0045     return (dev->untrusted == 0);
0046 }
0047 EXPORT_SYMBOL_GPL(pci_ats_supported);
0048
0049 /**
0050  * pci_enable_ats - enable the ATS capability
0051  * @dev: the PCI device
0052  * @ps: the IOMMU page shift
0053  *
0054  * Returns 0 on success, or negative on failure.
0055  */
0056 int pci_enable_ats(struct pci_dev *dev, int ps)
0057 {
0058     u16 ctrl;
0059     struct pci_dev *pdev;
0060
0061     if (!pci_ats_supported(dev))
0062         return -EINVAL;
0063
0064     if (WARN_ON(dev->ats_enabled))
0065         return -EBUSY;
0066
0067     if (ps < PCI_ATS_MIN_STU)
0068         return -EINVAL;
0069
0070     /*
0071      * Note that enabling ATS on a VF fails unless it's already enabled
0072      * with the same STU on the PF.
0073      */
0074     ctrl = PCI_ATS_CTRL_ENABLE;
0075     if (dev->is_virtfn) {
0076         pdev = pci_physfn(dev);
0077         if (pdev->ats_stu != ps)
0078             return -EINVAL;
0079     } else {
0080         dev->ats_stu = ps;
0081         ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
0082     }
0083     pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
0084
0085     dev->ats_enabled = 1;
0086     return 0;
0087 }
0088 EXPORT_SYMBOL_GPL(pci_enable_ats);
0089
0090 /**
0091  * pci_disable_ats - disable the ATS capability
0092  * @dev: the PCI device
0093  */
0094 void pci_disable_ats(struct pci_dev *dev)
0095 {
0096     u16 ctrl;
0097
0098     if (WARN_ON(!dev->ats_enabled))
0099         return;
0100
0101     pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
0102     ctrl &= ~PCI_ATS_CTRL_ENABLE;
0103     pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
0104
0105     dev->ats_enabled = 0;
0106 }
0107 EXPORT_SYMBOL_GPL(pci_disable_ats);
0108
0109 void pci_restore_ats_state(struct pci_dev *dev)
0110 {
0111     u16 ctrl;
0112
0113     if (!dev->ats_enabled)
0114         return;
0115
0116     ctrl = PCI_ATS_CTRL_ENABLE;
0117     if (!dev->is_virtfn)
0118         ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
0119     pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
0120 }
0121
0122 /**
0123  * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
0124  * @dev: the PCI device
0125  *
0126  * Returns the queue depth on success, or negative on failure.
0127  *
0128  * The ATS spec uses 0 in the Invalidate Queue Depth field to
0129  * indicate that the function can accept 32 Invalidate Request.
0130  * But here we use the `real' values (i.e. 1~32) for the Queue
0131  * Depth; and 0 indicates the function shares the Queue with
0132  * other functions (doesn't exclusively own a Queue).
0133  */
0134 int pci_ats_queue_depth(struct pci_dev *dev)
0135 {
0136     u16 cap;
0137
0138     if (!dev->ats_cap)
0139         return -EINVAL;
0140
0141     if (dev->is_virtfn)
0142         return 0;
0143
0144     pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
0145     return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;
0146 }
0147
0148 /**
0149  * pci_ats_page_aligned - Return Page Aligned Request bit status.
0150  * @pdev: the PCI device
0151  *
0152  * Returns 1, if the Untranslated Addresses generated by the device
0153  * are always aligned or 0 otherwise.
0154  *
0155  * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit
0156  * is set, it indicates the Untranslated Addresses generated by the
0157  * device are always aligned to a 4096 byte boundary.
0158  */
0159 int pci_ats_page_aligned(struct pci_dev *pdev)
0160 {
0161     u16 cap;
0162
0163     if (!pdev->ats_cap)
0164         return 0;
0165
0166     pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap);
0167
0168     if (cap & PCI_ATS_CAP_PAGE_ALIGNED)
0169         return 1;
0170
0171     return 0;
0172 }
0173
0174 #ifdef CONFIG_PCI_PRI
0175 void pci_pri_init(struct pci_dev *pdev)
0176 {
0177     u16 status;
0178
0179     pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
0180
0181     if (!pdev->pri_cap)
0182         return;
0183
0184     pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status);
0185     if (status & PCI_PRI_STATUS_PASID)
0186         pdev->pasid_required = 1;
0187 }
0188
0189 /**
0190  * pci_enable_pri - Enable PRI capability
0191  * @pdev: PCI device structure
0192  * @reqs: outstanding requests
0193  *
0194  * Returns 0 on success, negative value on error
0195  */
0196 int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
0197 {
0198     u16 control, status;
0199     u32 max_requests;
0200     int pri = pdev->pri_cap;
0201
0202     /*
0203      * VFs must not implement the PRI Capability.  If their PF
0204      * implements PRI, it is shared by the VFs, so if the PF PRI is
0205      * enabled, it is also enabled for the VF.
0206      */
0207     if (pdev->is_virtfn) {
0208         if (pci_physfn(pdev)->pri_enabled)
0209             return 0;
0210         return -EINVAL;
0211     }
0212
0213     if (WARN_ON(pdev->pri_enabled))
0214         return -EBUSY;
0215
0216     if (!pri)
0217         return -EINVAL;
0218
0219     pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status);
0220     if (!(status & PCI_PRI_STATUS_STOPPED))
0221         return -EBUSY;
0222
0223     pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests);
0224     reqs = min(max_requests, reqs);
0225     pdev->pri_reqs_alloc = reqs;
0226     pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
0227
0228     control = PCI_PRI_CTRL_ENABLE;
0229     pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
0230
0231     pdev->pri_enabled = 1;
0232
0233     return 0;
0234 }
0235
0236 /**
0237  * pci_disable_pri - Disable PRI capability
0238  * @pdev: PCI device structure
0239  *
0240  * Only clears the enabled-bit, regardless of its former value
0241  */
0242 void pci_disable_pri(struct pci_dev *pdev)
0243 {
0244     u16 control;
0245     int pri = pdev->pri_cap;
0246
0247     /* VFs share the PF PRI */
0248     if (pdev->is_virtfn)
0249         return;
0250
0251     if (WARN_ON(!pdev->pri_enabled))
0252         return;
0253
0254     if (!pri)
0255         return;
0256
0257     pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control);
0258     control &= ~PCI_PRI_CTRL_ENABLE;
0259     pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
0260
0261     pdev->pri_enabled = 0;
0262 }
0263 EXPORT_SYMBOL_GPL(pci_disable_pri);
0264
0265 /**
0266  * pci_restore_pri_state - Restore PRI
0267  * @pdev: PCI device structure
0268  */
0269 void pci_restore_pri_state(struct pci_dev *pdev)
0270 {
0271     u16 control = PCI_PRI_CTRL_ENABLE;
0272     u32 reqs = pdev->pri_reqs_alloc;
0273     int pri = pdev->pri_cap;
0274
0275     if (pdev->is_virtfn)
0276         return;
0277
0278     if (!pdev->pri_enabled)
0279         return;
0280
0281     if (!pri)
0282         return;
0283
0284     pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
0285     pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
0286 }
0287
0288 /**
0289  * pci_reset_pri - Resets device's PRI state
0290  * @pdev: PCI device structure
0291  *
0292  * The PRI capability must be disabled before this function is called.
0293  * Returns 0 on success, negative value on error.
0294  */
0295 int pci_reset_pri(struct pci_dev *pdev)
0296 {
0297     u16 control;
0298     int pri = pdev->pri_cap;
0299
0300     if (pdev->is_virtfn)
0301         return 0;
0302
0303     if (WARN_ON(pdev->pri_enabled))
0304         return -EBUSY;
0305
0306     if (!pri)
0307         return -EINVAL;
0308
0309     control = PCI_PRI_CTRL_RESET;
0310     pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
0311
0312     return 0;
0313 }
0314
0315 /**
0316  * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit
0317  *               status.
0318  * @pdev: PCI device structure
0319  *
0320  * Returns 1 if PASID is required in PRG Response Message, 0 otherwise.
0321  */
0322 int pci_prg_resp_pasid_required(struct pci_dev *pdev)
0323 {
0324     if (pdev->is_virtfn)
0325         pdev = pci_physfn(pdev);
0326
0327     return pdev->pasid_required;
0328 }
0329
0330 /**
0331  * pci_pri_supported - Check if PRI is supported.
0332  * @pdev: PCI device structure
0333  *
0334  * Returns true if PRI capability is present, false otherwise.
0335  */
0336 bool pci_pri_supported(struct pci_dev *pdev)
0337 {
0338     /* VFs share the PF PRI */
0339     if (pci_physfn(pdev)->pri_cap)
0340         return true;
0341     return false;
0342 }
0343 EXPORT_SYMBOL_GPL(pci_pri_supported);
0344 #endif /* CONFIG_PCI_PRI */
0345
0346 #ifdef CONFIG_PCI_PASID
0347 void pci_pasid_init(struct pci_dev *pdev)
0348 {
0349     pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
0350 }
0351
0352 /**
0353  * pci_enable_pasid - Enable the PASID capability
0354  * @pdev: PCI device structure
0355  * @features: Features to enable
0356  *
0357  * Returns 0 on success, negative value on error. This function checks
0358  * whether the features are actually supported by the device and returns
0359  * an error if not.
0360  */
0361 int pci_enable_pasid(struct pci_dev *pdev, int features)
0362 {
0363     u16 control, supported;
0364     int pasid = pdev->pasid_cap;
0365
0366     /*
0367      * VFs must not implement the PASID Capability, but if a PF
0368      * supports PASID, its VFs share the PF PASID configuration.
0369      */
0370     if (pdev->is_virtfn) {
0371         if (pci_physfn(pdev)->pasid_enabled)
0372             return 0;
0373         return -EINVAL;
0374     }
0375
0376     if (WARN_ON(pdev->pasid_enabled))
0377         return -EBUSY;
0378
0379     if (!pdev->eetlp_prefix_path && !pdev->pasid_no_tlp)
0380         return -EINVAL;
0381
0382     if (!pasid)
0383         return -EINVAL;
0384
0385     pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
0386     supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
0387
0388     /* User wants to enable anything unsupported? */
0389     if ((supported & features) != features)
0390         return -EINVAL;
0391
0392     control = PCI_PASID_CTRL_ENABLE | features;
0393     pdev->pasid_features = features;
0394
0395     pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
0396
0397     pdev->pasid_enabled = 1;
0398
0399     return 0;
0400 }
0401 EXPORT_SYMBOL_GPL(pci_enable_pasid);
0402
0403 /**
0404  * pci_disable_pasid - Disable the PASID capability
0405  * @pdev: PCI device structure
0406  */
0407 void pci_disable_pasid(struct pci_dev *pdev)
0408 {
0409     u16 control = 0;
0410     int pasid = pdev->pasid_cap;
0411
0412     /* VFs share the PF PASID configuration */
0413     if (pdev->is_virtfn)
0414         return;
0415
0416     if (WARN_ON(!pdev->pasid_enabled))
0417         return;
0418
0419     if (!pasid)
0420         return;
0421
0422     pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
0423
0424     pdev->pasid_enabled = 0;
0425 }
0426 EXPORT_SYMBOL_GPL(pci_disable_pasid);
0427
0428 /**
0429  * pci_restore_pasid_state - Restore PASID capabilities
0430  * @pdev: PCI device structure
0431  */
0432 void pci_restore_pasid_state(struct pci_dev *pdev)
0433 {
0434     u16 control;
0435     int pasid = pdev->pasid_cap;
0436
0437     if (pdev->is_virtfn)
0438         return;
0439
0440     if (!pdev->pasid_enabled)
0441         return;
0442
0443     if (!pasid)
0444         return;
0445
0446     control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features;
0447     pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
0448 }
0449
0450 /**
0451  * pci_pasid_features - Check which PASID features are supported
0452  * @pdev: PCI device structure
0453  *
0454  * Returns a negative value when no PASI capability is present.
0455  * Otherwise is returns a bitmask with supported features. Current
0456  * features reported are:
0457  * PCI_PASID_CAP_EXEC - Execute permission supported
0458  * PCI_PASID_CAP_PRIV - Privileged mode supported
0459  */
0460 int pci_pasid_features(struct pci_dev *pdev)
0461 {
0462     u16 supported;
0463     int pasid;
0464
0465     if (pdev->is_virtfn)
0466         pdev = pci_physfn(pdev);
0467
0468     pasid = pdev->pasid_cap;
0469     if (!pasid)
0470         return -EINVAL;
0471
0472     pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
0473
0474     supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
0475
0476     return supported;
0477 }
0478 EXPORT_SYMBOL_GPL(pci_pasid_features);
0479
0480 #define PASID_NUMBER_SHIFT  8
0481 #define PASID_NUMBER_MASK   (0x1f << PASID_NUMBER_SHIFT)
0482 /**
0483  * pci_max_pasids - Get maximum number of PASIDs supported by device
0484  * @pdev: PCI device structure
0485  *
0486  * Returns negative value when PASID capability is not present.
0487  * Otherwise it returns the number of supported PASIDs.
0488  */
0489 int pci_max_pasids(struct pci_dev *pdev)
0490 {
0491     u16 supported;
0492     int pasid;
0493
0494     if (pdev->is_virtfn)
0495         pdev = pci_physfn(pdev);
0496
0497     pasid = pdev->pasid_cap;
0498     if (!pasid)
0499         return -EINVAL;
0500
0501     pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
0502
0503     supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT;
0504
0505     return (1 << supported);
0506 }
0507 EXPORT_SYMBOL_GPL(pci_max_pasids);
0508 #endif /* CONFIG_PCI_PASID */