misc/cxl/cxllib.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright 2017 IBM Corp.
0004  */
0005
0006 #include <linux/hugetlb.h>
0007 #include <linux/sched/mm.h>
0008 #include <asm/opal-api.h>
0009 #include <asm/pnv-pci.h>
0010 #include <misc/cxllib.h>
0011
0012 #include "cxl.h"
0013
0014 #define CXL_INVALID_DRA                 ~0ull
0015 #define CXL_DUMMY_READ_SIZE             128
0016 #define CXL_DUMMY_READ_ALIGN            8
0017 #define CXL_CAPI_WINDOW_START           0x2000000000000ull
0018 #define CXL_CAPI_WINDOW_LOG_SIZE        48
0019 #define CXL_XSL_CONFIG_CURRENT_VERSION  CXL_XSL_CONFIG_VERSION1
0020
0021
0022 bool cxllib_slot_is_supported(struct pci_dev *dev, unsigned long flags)
0023 {
0024     int rc;
0025     u32 phb_index;
0026     u64 chip_id, capp_unit_id;
0027
0028     /* No flags currently supported */
0029     if (flags)
0030         return false;
0031
0032     if (!cpu_has_feature(CPU_FTR_HVMODE))
0033         return false;
0034
0035     if (!cxl_is_power9())
0036         return false;
0037
0038     if (cxl_slot_is_switched(dev))
0039         return false;
0040
0041     /* on p9, some pci slots are not connected to a CAPP unit */
0042     rc = cxl_calc_capp_routing(dev, &chip_id, &phb_index, &capp_unit_id);
0043     if (rc)
0044         return false;
0045
0046     return true;
0047 }
0048 EXPORT_SYMBOL_GPL(cxllib_slot_is_supported);
0049
0050 static DEFINE_MUTEX(dra_mutex);
0051 static u64 dummy_read_addr = CXL_INVALID_DRA;
0052
0053 static int allocate_dummy_read_buf(void)
0054 {
0055     u64 buf, vaddr;
0056     size_t buf_size;
0057
0058     /*
0059      * Dummy read buffer is 128-byte long, aligned on a
0060      * 256-byte boundary and we need the physical address.
0061      */
0062     buf_size = CXL_DUMMY_READ_SIZE + (1ull << CXL_DUMMY_READ_ALIGN);
0063     buf = (u64) kzalloc(buf_size, GFP_KERNEL);
0064     if (!buf)
0065         return -ENOMEM;
0066
0067     vaddr = (buf + (1ull << CXL_DUMMY_READ_ALIGN) - 1) &
0068                     (~0ull << CXL_DUMMY_READ_ALIGN);
0069
0070     WARN((vaddr + CXL_DUMMY_READ_SIZE) > (buf + buf_size),
0071         "Dummy read buffer alignment issue");
0072     dummy_read_addr = virt_to_phys((void *) vaddr);
0073     return 0;
0074 }
0075
0076 int cxllib_get_xsl_config(struct pci_dev *dev, struct cxllib_xsl_config *cfg)
0077 {
0078     int rc;
0079     u32 phb_index;
0080     u64 chip_id, capp_unit_id;
0081
0082     if (!cpu_has_feature(CPU_FTR_HVMODE))
0083         return -EINVAL;
0084
0085     mutex_lock(&dra_mutex);
0086     if (dummy_read_addr == CXL_INVALID_DRA) {
0087         rc = allocate_dummy_read_buf();
0088         if (rc) {
0089             mutex_unlock(&dra_mutex);
0090             return rc;
0091         }
0092     }
0093     mutex_unlock(&dra_mutex);
0094
0095     rc = cxl_calc_capp_routing(dev, &chip_id, &phb_index, &capp_unit_id);
0096     if (rc)
0097         return rc;
0098
0099     rc = cxl_get_xsl9_dsnctl(dev, capp_unit_id, &cfg->dsnctl);
0100     if (rc)
0101         return rc;
0102
0103     cfg->version  = CXL_XSL_CONFIG_CURRENT_VERSION;
0104     cfg->log_bar_size = CXL_CAPI_WINDOW_LOG_SIZE;
0105     cfg->bar_addr = CXL_CAPI_WINDOW_START;
0106     cfg->dra = dummy_read_addr;
0107     return 0;
0108 }
0109 EXPORT_SYMBOL_GPL(cxllib_get_xsl_config);
0110
0111 int cxllib_switch_phb_mode(struct pci_dev *dev, enum cxllib_mode mode,
0112             unsigned long flags)
0113 {
0114     int rc = 0;
0115
0116     if (!cpu_has_feature(CPU_FTR_HVMODE))
0117         return -EINVAL;
0118
0119     switch (mode) {
0120     case CXL_MODE_PCI:
0121         /*
0122          * We currently don't support going back to PCI mode
0123          * However, we'll turn the invalidations off, so that
0124          * the firmware doesn't have to ack them and can do
0125          * things like reset, etc.. with no worries.
0126          * So always return EPERM (can't go back to PCI) or
0127          * EBUSY if we couldn't even turn off snooping
0128          */
0129         rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_OFF);
0130         if (rc)
0131             rc = -EBUSY;
0132         else
0133             rc = -EPERM;
0134         break;
0135     case CXL_MODE_CXL:
0136         /* DMA only supported on TVT1 for the time being */
0137         if (flags != CXL_MODE_DMA_TVT1)
0138             return -EINVAL;
0139         rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_DMA_TVT1);
0140         if (rc)
0141             return rc;
0142         rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_ON);
0143         break;
0144     default:
0145         rc = -EINVAL;
0146     }
0147     return rc;
0148 }
0149 EXPORT_SYMBOL_GPL(cxllib_switch_phb_mode);
0150
0151 /*
0152  * When switching the PHB to capi mode, the TVT#1 entry for
0153  * the Partitionable Endpoint is set in bypass mode, like
0154  * in PCI mode.
0155  * Configure the device dma to use TVT#1, which is done
0156  * by calling dma_set_mask() with a mask large enough.
0157  */
0158 int cxllib_set_device_dma(struct pci_dev *dev, unsigned long flags)
0159 {
0160     int rc;
0161
0162     if (flags)
0163         return -EINVAL;
0164
0165     rc = dma_set_mask(&dev->dev, DMA_BIT_MASK(64));
0166     return rc;
0167 }
0168 EXPORT_SYMBOL_GPL(cxllib_set_device_dma);
0169
0170 int cxllib_get_PE_attributes(struct task_struct *task,
0171                  unsigned long translation_mode,
0172                  struct cxllib_pe_attributes *attr)
0173 {
0174     if (translation_mode != CXL_TRANSLATED_MODE &&
0175         translation_mode != CXL_REAL_MODE)
0176         return -EINVAL;
0177
0178     attr->sr = cxl_calculate_sr(false,
0179                 task == NULL,
0180                 translation_mode == CXL_REAL_MODE,
0181                 true);
0182     attr->lpid = mfspr(SPRN_LPID);
0183     if (task) {
0184         struct mm_struct *mm = get_task_mm(task);
0185         if (mm == NULL)
0186             return -EINVAL;
0187         /*
0188          * Caller is keeping a reference on mm_users for as long
0189          * as XSL uses the memory context
0190          */
0191         attr->pid = mm->context.id;
0192         mmput(mm);
0193         attr->tid = task->thread.tidr;
0194     } else {
0195         attr->pid = 0;
0196         attr->tid = 0;
0197     }
0198     return 0;
0199 }
0200 EXPORT_SYMBOL_GPL(cxllib_get_PE_attributes);
0201
0202 static int get_vma_info(struct mm_struct *mm, u64 addr,
0203             u64 *vma_start, u64 *vma_end,
0204             unsigned long *page_size)
0205 {
0206     struct vm_area_struct *vma = NULL;
0207     int rc = 0;
0208
0209     mmap_read_lock(mm);
0210
0211     vma = find_vma(mm, addr);
0212     if (!vma) {
0213         rc = -EFAULT;
0214         goto out;
0215     }
0216     *page_size = vma_kernel_pagesize(vma);
0217     *vma_start = vma->vm_start;
0218     *vma_end = vma->vm_end;
0219 out:
0220     mmap_read_unlock(mm);
0221     return rc;
0222 }
0223
0224 int cxllib_handle_fault(struct mm_struct *mm, u64 addr, u64 size, u64 flags)
0225 {
0226     int rc;
0227     u64 dar, vma_start, vma_end;
0228     unsigned long page_size;
0229
0230     if (mm == NULL)
0231         return -EFAULT;
0232
0233     /*
0234      * The buffer we have to process can extend over several pages
0235      * and may also cover several VMAs.
0236      * We iterate over all the pages. The page size could vary
0237      * between VMAs.
0238      */
0239     rc = get_vma_info(mm, addr, &vma_start, &vma_end, &page_size);
0240     if (rc)
0241         return rc;
0242
0243     for (dar = (addr & ~(page_size - 1)); dar < (addr + size);
0244          dar += page_size) {
0245         if (dar < vma_start || dar >= vma_end) {
0246             /*
0247              * We don't hold mm->mmap_lock while iterating, since
0248              * the lock is required by one of the lower-level page
0249              * fault processing functions and it could
0250              * create a deadlock.
0251              *
0252              * It means the VMAs can be altered between 2
0253              * loop iterations and we could theoretically
0254              * miss a page (however unlikely). But that's
0255              * not really a problem, as the driver will
0256              * retry access, get another page fault on the
0257              * missing page and call us again.
0258              */
0259             rc = get_vma_info(mm, dar, &vma_start, &vma_end,
0260                     &page_size);
0261             if (rc)
0262                 return rc;
0263         }
0264
0265         rc = cxl_handle_mm_fault(mm, flags, dar);
0266         if (rc)
0267             return -EFAULT;
0268     }
0269     return 0;
0270 }
0271 EXPORT_SYMBOL_GPL(cxllib_handle_fault);