amd/amdkfd/kfd_iommu.c

0001 // SPDX-License-Identifier: GPL-2.0 OR MIT
0002 /*
0003  * Copyright 2018-2022 Advanced Micro Devices, Inc.
0004  *
0005  * Permission is hereby granted, free of charge, to any person obtaining a
0006  * copy of this software and associated documentation files (the "Software"),
0007  * to deal in the Software without restriction, including without limitation
0008  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
0009  * and/or sell copies of the Software, and to permit persons to whom the
0010  * Software is furnished to do so, subject to the following conditions:
0011  *
0012  * The above copyright notice and this permission notice shall be included in
0013  * all copies or substantial portions of the Software.
0014  *
0015  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
0016  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
0017  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
0018  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
0019  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
0020  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
0021  * OTHER DEALINGS IN THE SOFTWARE.
0022  */
0023
0024 #include <linux/kconfig.h>
0025
0026 #if IS_REACHABLE(CONFIG_AMD_IOMMU_V2)
0027
0028 #include <linux/printk.h>
0029 #include <linux/device.h>
0030 #include <linux/slab.h>
0031 #include <linux/pci.h>
0032 #include <linux/amd-iommu.h>
0033 #include "kfd_priv.h"
0034 #include "kfd_topology.h"
0035 #include "kfd_iommu.h"
0036
0037 static const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
0038                     AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
0039                     AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
0040
0041 /** kfd_iommu_check_device - Check whether IOMMU is available for device
0042  */
0043 int kfd_iommu_check_device(struct kfd_dev *kfd)
0044 {
0045     struct amd_iommu_device_info iommu_info;
0046     int err;
0047
0048     if (!kfd->use_iommu_v2)
0049         return -ENODEV;
0050
0051     iommu_info.flags = 0;
0052     err = amd_iommu_device_info(kfd->pdev, &iommu_info);
0053     if (err)
0054         return err;
0055
0056     if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags)
0057         return -ENODEV;
0058
0059     return 0;
0060 }
0061
0062 /** kfd_iommu_device_init - Initialize IOMMU for device
0063  */
0064 int kfd_iommu_device_init(struct kfd_dev *kfd)
0065 {
0066     struct amd_iommu_device_info iommu_info;
0067     unsigned int pasid_limit;
0068     int err;
0069
0070     if (!kfd->use_iommu_v2)
0071         return 0;
0072
0073     iommu_info.flags = 0;
0074     err = amd_iommu_device_info(kfd->pdev, &iommu_info);
0075     if (err < 0) {
0076         dev_err(kfd_device,
0077             "error getting iommu info. is the iommu enabled?\n");
0078         return -ENODEV;
0079     }
0080
0081     if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
0082         dev_err(kfd_device,
0083             "error required iommu flags ats %i, pri %i, pasid %i\n",
0084                (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
0085                (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
0086                (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
0087                                     != 0);
0088         return -ENODEV;
0089     }
0090
0091     pasid_limit = min_t(unsigned int,
0092             (unsigned int)(1 << kfd->device_info.max_pasid_bits),
0093             iommu_info.max_pasids);
0094
0095     if (!kfd_set_pasid_limit(pasid_limit)) {
0096         dev_err(kfd_device, "error setting pasid limit\n");
0097         return -EBUSY;
0098     }
0099
0100     return 0;
0101 }
0102
0103 /** kfd_iommu_bind_process_to_device - Have the IOMMU bind a process
0104  *
0105  * Binds the given process to the given device using its PASID. This
0106  * enables IOMMUv2 address translation for the process on the device.
0107  *
0108  * This function assumes that the process mutex is held.
0109  */
0110 int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd)
0111 {
0112     struct kfd_dev *dev = pdd->dev;
0113     struct kfd_process *p = pdd->process;
0114     int err;
0115
0116     if (!dev->use_iommu_v2 || pdd->bound == PDD_BOUND)
0117         return 0;
0118
0119     if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
0120         pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
0121         return -EINVAL;
0122     }
0123
0124     err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
0125     if (!err)
0126         pdd->bound = PDD_BOUND;
0127
0128     return err;
0129 }
0130
0131 /** kfd_iommu_unbind_process - Unbind process from all devices
0132  *
0133  * This removes all IOMMU device bindings of the process. To be used
0134  * before process termination.
0135  */
0136 void kfd_iommu_unbind_process(struct kfd_process *p)
0137 {
0138     int i;
0139
0140     for (i = 0; i < p->n_pdds; i++)
0141         if (p->pdds[i]->bound == PDD_BOUND)
0142             amd_iommu_unbind_pasid(p->pdds[i]->dev->pdev, p->pasid);
0143 }
0144
0145 /* Callback for process shutdown invoked by the IOMMU driver */
0146 static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, u32 pasid)
0147 {
0148     struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
0149     struct kfd_process *p;
0150     struct kfd_process_device *pdd;
0151
0152     if (!dev)
0153         return;
0154
0155     /*
0156      * Look for the process that matches the pasid. If there is no such
0157      * process, we either released it in amdkfd's own notifier, or there
0158      * is a bug. Unfortunately, there is no way to tell...
0159      */
0160     p = kfd_lookup_process_by_pasid(pasid);
0161     if (!p)
0162         return;
0163
0164     pr_debug("Unbinding process 0x%x from IOMMU\n", pasid);
0165
0166     mutex_lock(&p->mutex);
0167
0168     pdd = kfd_get_process_device_data(dev, p);
0169     if (pdd)
0170         /* For GPU relying on IOMMU, we need to dequeue here
0171          * when PASID is still bound.
0172          */
0173         kfd_process_dequeue_from_device(pdd);
0174
0175     mutex_unlock(&p->mutex);
0176
0177     kfd_unref_process(p);
0178 }
0179
0180 /* This function called by IOMMU driver on PPR failure */
0181 static int iommu_invalid_ppr_cb(struct pci_dev *pdev, u32 pasid,
0182                 unsigned long address, u16 flags)
0183 {
0184     struct kfd_dev *dev;
0185
0186     dev_warn_ratelimited(kfd_device,
0187             "Invalid PPR device %x:%x.%x pasid 0x%x address 0x%lX flags 0x%X",
0188             pdev->bus->number,
0189             PCI_SLOT(pdev->devfn),
0190             PCI_FUNC(pdev->devfn),
0191             pasid,
0192             address,
0193             flags);
0194
0195     dev = kfd_device_by_pci_dev(pdev);
0196     if (!WARN_ON(!dev))
0197         kfd_signal_iommu_event(dev, pasid, address,
0198             flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
0199
0200     return AMD_IOMMU_INV_PRI_RSP_INVALID;
0201 }
0202
0203 /*
0204  * Bind processes do the device that have been temporarily unbound
0205  * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
0206  */
0207 static int kfd_bind_processes_to_device(struct kfd_dev *kfd)
0208 {
0209     struct kfd_process_device *pdd;
0210     struct kfd_process *p;
0211     unsigned int temp;
0212     int err = 0;
0213
0214     int idx = srcu_read_lock(&kfd_processes_srcu);
0215
0216     hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
0217         mutex_lock(&p->mutex);
0218         pdd = kfd_get_process_device_data(kfd, p);
0219
0220         if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
0221             mutex_unlock(&p->mutex);
0222             continue;
0223         }
0224
0225         err = amd_iommu_bind_pasid(kfd->pdev, p->pasid,
0226                 p->lead_thread);
0227         if (err < 0) {
0228             pr_err("Unexpected pasid 0x%x binding failure\n",
0229                     p->pasid);
0230             mutex_unlock(&p->mutex);
0231             break;
0232         }
0233
0234         pdd->bound = PDD_BOUND;
0235         mutex_unlock(&p->mutex);
0236     }
0237
0238     srcu_read_unlock(&kfd_processes_srcu, idx);
0239
0240     return err;
0241 }
0242
0243 /*
0244  * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
0245  * processes will be restored to PDD_BOUND state in
0246  * kfd_bind_processes_to_device.
0247  */
0248 static void kfd_unbind_processes_from_device(struct kfd_dev *kfd)
0249 {
0250     struct kfd_process_device *pdd;
0251     struct kfd_process *p;
0252     unsigned int temp;
0253
0254     int idx = srcu_read_lock(&kfd_processes_srcu);
0255
0256     hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
0257         mutex_lock(&p->mutex);
0258         pdd = kfd_get_process_device_data(kfd, p);
0259
0260         if (WARN_ON(!pdd)) {
0261             mutex_unlock(&p->mutex);
0262             continue;
0263         }
0264
0265         if (pdd->bound == PDD_BOUND)
0266             pdd->bound = PDD_BOUND_SUSPENDED;
0267         mutex_unlock(&p->mutex);
0268     }
0269
0270     srcu_read_unlock(&kfd_processes_srcu, idx);
0271 }
0272
0273 /** kfd_iommu_suspend - Prepare IOMMU for suspend
0274  *
0275  * This unbinds processes from the device and disables the IOMMU for
0276  * the device.
0277  */
0278 void kfd_iommu_suspend(struct kfd_dev *kfd)
0279 {
0280     if (!kfd->use_iommu_v2)
0281         return;
0282
0283     kfd_unbind_processes_from_device(kfd);
0284
0285     amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
0286     amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
0287     amd_iommu_free_device(kfd->pdev);
0288 }
0289
0290 /** kfd_iommu_resume - Restore IOMMU after resume
0291  *
0292  * This reinitializes the IOMMU for the device and re-binds previously
0293  * suspended processes to the device.
0294  */
0295 int kfd_iommu_resume(struct kfd_dev *kfd)
0296 {
0297     unsigned int pasid_limit;
0298     int err;
0299
0300     if (!kfd->use_iommu_v2)
0301         return 0;
0302
0303     pasid_limit = kfd_get_pasid_limit();
0304
0305     err = amd_iommu_init_device(kfd->pdev, pasid_limit);
0306     if (err)
0307         return -ENXIO;
0308
0309     amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
0310                     iommu_pasid_shutdown_callback);
0311     amd_iommu_set_invalid_ppr_cb(kfd->pdev,
0312                      iommu_invalid_ppr_cb);
0313
0314     err = kfd_bind_processes_to_device(kfd);
0315     if (err) {
0316         amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
0317         amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
0318         amd_iommu_free_device(kfd->pdev);
0319         return err;
0320     }
0321
0322     return 0;
0323 }
0324
0325 /** kfd_iommu_add_perf_counters - Add IOMMU performance counters to topology
0326  */
0327 int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev)
0328 {
0329     struct kfd_perf_properties *props;
0330
0331     if (!(kdev->node_props.capability & HSA_CAP_ATS_PRESENT))
0332         return 0;
0333
0334     if (!amd_iommu_pc_supported())
0335         return 0;
0336
0337     props = kfd_alloc_struct(props);
0338     if (!props)
0339         return -ENOMEM;
0340     strcpy(props->block_name, "iommu");
0341     props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
0342         amd_iommu_pc_get_max_counters(0); /* assume one iommu */
0343     list_add_tail(&props->list, &kdev->perf_props);
0344
0345     return 0;
0346 }
0347
0348 #endif