x86/kernel/pci-dma.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #include <linux/dma-map-ops.h>
0003 #include <linux/dma-direct.h>
0004 #include <linux/iommu.h>
0005 #include <linux/dmar.h>
0006 #include <linux/export.h>
0007 #include <linux/memblock.h>
0008 #include <linux/gfp.h>
0009 #include <linux/pci.h>
0010 #include <linux/amd-iommu.h>
0011
0012 #include <asm/proto.h>
0013 #include <asm/dma.h>
0014 #include <asm/iommu.h>
0015 #include <asm/gart.h>
0016 #include <asm/x86_init.h>
0017
0018 #include <xen/xen.h>
0019 #include <xen/swiotlb-xen.h>
0020
0021 static bool disable_dac_quirk __read_mostly;
0022
0023 const struct dma_map_ops *dma_ops;
0024 EXPORT_SYMBOL(dma_ops);
0025
0026 #ifdef CONFIG_IOMMU_DEBUG
0027 int panic_on_overflow __read_mostly = 1;
0028 int force_iommu __read_mostly = 1;
0029 #else
0030 int panic_on_overflow __read_mostly = 0;
0031 int force_iommu __read_mostly = 0;
0032 #endif
0033
0034 int iommu_merge __read_mostly = 0;
0035
0036 int no_iommu __read_mostly;
0037 /* Set this to 1 if there is a HW IOMMU in the system */
0038 int iommu_detected __read_mostly = 0;
0039
0040 #ifdef CONFIG_SWIOTLB
0041 bool x86_swiotlb_enable;
0042 static unsigned int x86_swiotlb_flags;
0043
0044 static void __init pci_swiotlb_detect(void)
0045 {
0046     /* don't initialize swiotlb if iommu=off (no_iommu=1) */
0047     if (!no_iommu && max_possible_pfn > MAX_DMA32_PFN)
0048         x86_swiotlb_enable = true;
0049
0050     /*
0051      * Set swiotlb to 1 so that bounce buffers are allocated and used for
0052      * devices that can't support DMA to encrypted memory.
0053      */
0054     if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT))
0055         x86_swiotlb_enable = true;
0056
0057     /*
0058      * Guest with guest memory encryption currently perform all DMA through
0059      * bounce buffers as the hypervisor can't access arbitrary VM memory
0060      * that is not explicitly shared with it.
0061      */
0062     if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
0063         x86_swiotlb_enable = true;
0064         x86_swiotlb_flags |= SWIOTLB_FORCE;
0065     }
0066 }
0067 #else
0068 static inline void __init pci_swiotlb_detect(void)
0069 {
0070 }
0071 #define x86_swiotlb_flags 0
0072 #endif /* CONFIG_SWIOTLB */
0073
0074 #ifdef CONFIG_SWIOTLB_XEN
0075 static void __init pci_xen_swiotlb_init(void)
0076 {
0077     if (!xen_initial_domain() && !x86_swiotlb_enable)
0078         return;
0079     x86_swiotlb_enable = true;
0080     x86_swiotlb_flags |= SWIOTLB_ANY;
0081     swiotlb_init_remap(true, x86_swiotlb_flags, xen_swiotlb_fixup);
0082     dma_ops = &xen_swiotlb_dma_ops;
0083     if (IS_ENABLED(CONFIG_PCI))
0084         pci_request_acs();
0085 }
0086
0087 int pci_xen_swiotlb_init_late(void)
0088 {
0089     if (dma_ops == &xen_swiotlb_dma_ops)
0090         return 0;
0091
0092     /* we can work with the default swiotlb */
0093     if (!io_tlb_default_mem.nslabs) {
0094         int rc = swiotlb_init_late(swiotlb_size_or_default(),
0095                        GFP_KERNEL, xen_swiotlb_fixup);
0096         if (rc < 0)
0097             return rc;
0098     }
0099
0100     /* XXX: this switches the dma ops under live devices! */
0101     dma_ops = &xen_swiotlb_dma_ops;
0102     if (IS_ENABLED(CONFIG_PCI))
0103         pci_request_acs();
0104     return 0;
0105 }
0106 EXPORT_SYMBOL_GPL(pci_xen_swiotlb_init_late);
0107 #else
0108 static inline void __init pci_xen_swiotlb_init(void)
0109 {
0110 }
0111 #endif /* CONFIG_SWIOTLB_XEN */
0112
0113 void __init pci_iommu_alloc(void)
0114 {
0115     if (xen_pv_domain()) {
0116         pci_xen_swiotlb_init();
0117         return;
0118     }
0119     pci_swiotlb_detect();
0120     gart_iommu_hole_init();
0121     amd_iommu_detect();
0122     detect_intel_iommu();
0123     swiotlb_init(x86_swiotlb_enable, x86_swiotlb_flags);
0124 }
0125
0126 /*
0127  * See <Documentation/x86/x86_64/boot-options.rst> for the iommu kernel
0128  * parameter documentation.
0129  */
0130 static __init int iommu_setup(char *p)
0131 {
0132     iommu_merge = 1;
0133
0134     if (!p)
0135         return -EINVAL;
0136
0137     while (*p) {
0138         if (!strncmp(p, "off", 3))
0139             no_iommu = 1;
0140         /* gart_parse_options has more force support */
0141         if (!strncmp(p, "force", 5))
0142             force_iommu = 1;
0143         if (!strncmp(p, "noforce", 7)) {
0144             iommu_merge = 0;
0145             force_iommu = 0;
0146         }
0147
0148         if (!strncmp(p, "biomerge", 8)) {
0149             iommu_merge = 1;
0150             force_iommu = 1;
0151         }
0152         if (!strncmp(p, "panic", 5))
0153             panic_on_overflow = 1;
0154         if (!strncmp(p, "nopanic", 7))
0155             panic_on_overflow = 0;
0156         if (!strncmp(p, "merge", 5)) {
0157             iommu_merge = 1;
0158             force_iommu = 1;
0159         }
0160         if (!strncmp(p, "nomerge", 7))
0161             iommu_merge = 0;
0162         if (!strncmp(p, "forcesac", 8))
0163             pr_warn("forcesac option ignored.\n");
0164         if (!strncmp(p, "allowdac", 8))
0165             pr_warn("allowdac option ignored.\n");
0166         if (!strncmp(p, "nodac", 5))
0167             pr_warn("nodac option ignored.\n");
0168         if (!strncmp(p, "usedac", 6)) {
0169             disable_dac_quirk = true;
0170             return 1;
0171         }
0172 #ifdef CONFIG_SWIOTLB
0173         if (!strncmp(p, "soft", 4))
0174             x86_swiotlb_enable = true;
0175 #endif
0176         if (!strncmp(p, "pt", 2))
0177             iommu_set_default_passthrough(true);
0178         if (!strncmp(p, "nopt", 4))
0179             iommu_set_default_translated(true);
0180
0181         gart_parse_options(p);
0182
0183         p += strcspn(p, ",");
0184         if (*p == ',')
0185             ++p;
0186     }
0187     return 0;
0188 }
0189 early_param("iommu", iommu_setup);
0190
0191 static int __init pci_iommu_init(void)
0192 {
0193     x86_init.iommu.iommu_init();
0194
0195 #ifdef CONFIG_SWIOTLB
0196     /* An IOMMU turned us off. */
0197     if (x86_swiotlb_enable) {
0198         pr_info("PCI-DMA: Using software bounce buffering for IO (SWIOTLB)\n");
0199         swiotlb_print_info();
0200     } else {
0201         swiotlb_exit();
0202     }
0203 #endif
0204
0205     return 0;
0206 }
0207 /* Must execute after PCI subsystem */
0208 rootfs_initcall(pci_iommu_init);
0209
0210 #ifdef CONFIG_PCI
0211 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
0212
0213 static int via_no_dac_cb(struct pci_dev *pdev, void *data)
0214 {
0215     pdev->dev.bus_dma_limit = DMA_BIT_MASK(32);
0216     return 0;
0217 }
0218
0219 static void via_no_dac(struct pci_dev *dev)
0220 {
0221     if (!disable_dac_quirk) {
0222         dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
0223         pci_walk_bus(dev->subordinate, via_no_dac_cb, NULL);
0224     }
0225 }
0226 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID,
0227                 PCI_CLASS_BRIDGE_PCI, 8, via_no_dac);
0228 #endif