Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Access to PCI I/O memory from user space programs.
0004  *
0005  * Copyright IBM Corp. 2014
0006  * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com>
0007  */
0008 #include <linux/kernel.h>
0009 #include <linux/syscalls.h>
0010 #include <linux/init.h>
0011 #include <linux/mm.h>
0012 #include <linux/errno.h>
0013 #include <linux/pci.h>
0014 #include <asm/asm-extable.h>
0015 #include <asm/pci_io.h>
0016 #include <asm/pci_debug.h>
0017 
0018 static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset)
0019 {
0020     struct {
0021         u64 offset;
0022         u8 cc;
0023         u8 status;
0024     } data = {offset, cc, status};
0025 
0026     zpci_err_hex(&data, sizeof(data));
0027 }
0028 
0029 static inline int __pcistb_mio_inuser(
0030         void __iomem *ioaddr, const void __user *src,
0031         u64 len, u8 *status)
0032 {
0033     int cc = -ENXIO;
0034 
0035     asm volatile (
0036         "       sacf 256\n"
0037         "0:     .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n"
0038         "1:     ipm     %[cc]\n"
0039         "       srl     %[cc],28\n"
0040         "2:     sacf 768\n"
0041         EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
0042         : [cc] "+d" (cc), [len] "+d" (len)
0043         : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src))
0044         : "cc", "memory");
0045     *status = len >> 24 & 0xff;
0046     return cc;
0047 }
0048 
0049 static inline int __pcistg_mio_inuser(
0050         void __iomem *ioaddr, const void __user *src,
0051         u64 ulen, u8 *status)
0052 {
0053     union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
0054     int cc = -ENXIO;
0055     u64 val = 0;
0056     u64 cnt = ulen;
0057     u8 tmp;
0058 
0059     /*
0060      * copy 0 < @len <= 8 bytes from @src into the right most bytes of
0061      * a register, then store it to PCI at @ioaddr while in secondary
0062      * address space. pcistg then uses the user mappings.
0063      */
0064     asm volatile (
0065         "       sacf    256\n"
0066         "0:     llgc    %[tmp],0(%[src])\n"
0067         "       sllg    %[val],%[val],8\n"
0068         "       aghi    %[src],1\n"
0069         "       ogr     %[val],%[tmp]\n"
0070         "       brctg   %[cnt],0b\n"
0071         "1:     .insn   rre,0xb9d40000,%[val],%[ioaddr_len]\n"
0072         "2:     ipm     %[cc]\n"
0073         "       srl     %[cc],28\n"
0074         "3:     sacf    768\n"
0075         EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b)
0076         :
0077         [src] "+a" (src), [cnt] "+d" (cnt),
0078         [val] "+d" (val), [tmp] "=d" (tmp),
0079         [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair)
0080         :: "cc", "memory");
0081     *status = ioaddr_len.odd >> 24 & 0xff;
0082 
0083     /* did we read everything from user memory? */
0084     if (!cc && cnt != 0)
0085         cc = -EFAULT;
0086 
0087     return cc;
0088 }
0089 
0090 static inline int __memcpy_toio_inuser(void __iomem *dst,
0091                    const void __user *src, size_t n)
0092 {
0093     int size, rc = 0;
0094     u8 status = 0;
0095 
0096     if (!src)
0097         return -EINVAL;
0098 
0099     while (n > 0) {
0100         size = zpci_get_max_write_size((u64 __force) dst,
0101                            (u64 __force) src, n,
0102                            ZPCI_MAX_WRITE_SIZE);
0103         if (size > 8) /* main path */
0104             rc = __pcistb_mio_inuser(dst, src, size, &status);
0105         else
0106             rc = __pcistg_mio_inuser(dst, src, size, &status);
0107         if (rc)
0108             break;
0109         src += size;
0110         dst += size;
0111         n -= size;
0112     }
0113     if (rc)
0114         zpci_err_mmio(rc, status, (__force u64) dst);
0115     return rc;
0116 }
0117 
0118 SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr,
0119         const void __user *, user_buffer, size_t, length)
0120 {
0121     u8 local_buf[64];
0122     void __iomem *io_addr;
0123     void *buf;
0124     struct vm_area_struct *vma;
0125     pte_t *ptep;
0126     spinlock_t *ptl;
0127     long ret;
0128 
0129     if (!zpci_is_enabled())
0130         return -ENODEV;
0131 
0132     if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
0133         return -EINVAL;
0134 
0135     /*
0136      * We only support write access to MIO capable devices if we are on
0137      * a MIO enabled system. Otherwise we would have to check for every
0138      * address if it is a special ZPCI_ADDR and would have to do
0139      * a pfn lookup which we don't need for MIO capable devices.  Currently
0140      * ISM devices are the only devices without MIO support and there is no
0141      * known need for accessing these from userspace.
0142      */
0143     if (static_branch_likely(&have_mio)) {
0144         ret = __memcpy_toio_inuser((void  __iomem *) mmio_addr,
0145                     user_buffer,
0146                     length);
0147         return ret;
0148     }
0149 
0150     if (length > 64) {
0151         buf = kmalloc(length, GFP_KERNEL);
0152         if (!buf)
0153             return -ENOMEM;
0154     } else
0155         buf = local_buf;
0156 
0157     ret = -EFAULT;
0158     if (copy_from_user(buf, user_buffer, length))
0159         goto out_free;
0160 
0161     mmap_read_lock(current->mm);
0162     ret = -EINVAL;
0163     vma = vma_lookup(current->mm, mmio_addr);
0164     if (!vma)
0165         goto out_unlock_mmap;
0166     if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
0167         goto out_unlock_mmap;
0168     ret = -EACCES;
0169     if (!(vma->vm_flags & VM_WRITE))
0170         goto out_unlock_mmap;
0171 
0172     ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
0173     if (ret)
0174         goto out_unlock_mmap;
0175 
0176     io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
0177             (mmio_addr & ~PAGE_MASK));
0178 
0179     if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE)
0180         goto out_unlock_pt;
0181 
0182     ret = zpci_memcpy_toio(io_addr, buf, length);
0183 out_unlock_pt:
0184     pte_unmap_unlock(ptep, ptl);
0185 out_unlock_mmap:
0186     mmap_read_unlock(current->mm);
0187 out_free:
0188     if (buf != local_buf)
0189         kfree(buf);
0190     return ret;
0191 }
0192 
0193 static inline int __pcilg_mio_inuser(
0194         void __user *dst, const void __iomem *ioaddr,
0195         u64 ulen, u8 *status)
0196 {
0197     union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen};
0198     u64 cnt = ulen;
0199     int shift = ulen * 8;
0200     int cc = -ENXIO;
0201     u64 val, tmp;
0202 
0203     /*
0204      * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in
0205      * user space) into a register using pcilg then store these bytes at
0206      * user address @dst
0207      */
0208     asm volatile (
0209         "       sacf    256\n"
0210         "0:     .insn   rre,0xb9d60000,%[val],%[ioaddr_len]\n"
0211         "1:     ipm     %[cc]\n"
0212         "       srl     %[cc],28\n"
0213         "       ltr     %[cc],%[cc]\n"
0214         "       jne     4f\n"
0215         "2:     ahi     %[shift],-8\n"
0216         "       srlg    %[tmp],%[val],0(%[shift])\n"
0217         "3:     stc     %[tmp],0(%[dst])\n"
0218         "       aghi    %[dst],1\n"
0219         "       brctg   %[cnt],2b\n"
0220         "4:     sacf    768\n"
0221         EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b)
0222         :
0223         [ioaddr_len] "+&d" (ioaddr_len.pair),
0224         [cc] "+d" (cc), [val] "=d" (val),
0225         [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp),
0226         [shift] "+d" (shift)
0227         :: "cc", "memory");
0228 
0229     /* did we write everything to the user space buffer? */
0230     if (!cc && cnt != 0)
0231         cc = -EFAULT;
0232 
0233     *status = ioaddr_len.odd >> 24 & 0xff;
0234     return cc;
0235 }
0236 
0237 static inline int __memcpy_fromio_inuser(void __user *dst,
0238                      const void __iomem *src,
0239                      unsigned long n)
0240 {
0241     int size, rc = 0;
0242     u8 status;
0243 
0244     while (n > 0) {
0245         size = zpci_get_max_write_size((u64 __force) src,
0246                            (u64 __force) dst, n,
0247                            ZPCI_MAX_READ_SIZE);
0248         rc = __pcilg_mio_inuser(dst, src, size, &status);
0249         if (rc)
0250             break;
0251         src += size;
0252         dst += size;
0253         n -= size;
0254     }
0255     if (rc)
0256         zpci_err_mmio(rc, status, (__force u64) dst);
0257     return rc;
0258 }
0259 
0260 SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr,
0261         void __user *, user_buffer, size_t, length)
0262 {
0263     u8 local_buf[64];
0264     void __iomem *io_addr;
0265     void *buf;
0266     struct vm_area_struct *vma;
0267     pte_t *ptep;
0268     spinlock_t *ptl;
0269     long ret;
0270 
0271     if (!zpci_is_enabled())
0272         return -ENODEV;
0273 
0274     if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length)
0275         return -EINVAL;
0276 
0277     /*
0278      * We only support read access to MIO capable devices if we are on
0279      * a MIO enabled system. Otherwise we would have to check for every
0280      * address if it is a special ZPCI_ADDR and would have to do
0281      * a pfn lookup which we don't need for MIO capable devices.  Currently
0282      * ISM devices are the only devices without MIO support and there is no
0283      * known need for accessing these from userspace.
0284      */
0285     if (static_branch_likely(&have_mio)) {
0286         ret = __memcpy_fromio_inuser(
0287                 user_buffer, (const void __iomem *)mmio_addr,
0288                 length);
0289         return ret;
0290     }
0291 
0292     if (length > 64) {
0293         buf = kmalloc(length, GFP_KERNEL);
0294         if (!buf)
0295             return -ENOMEM;
0296     } else {
0297         buf = local_buf;
0298     }
0299 
0300     mmap_read_lock(current->mm);
0301     ret = -EINVAL;
0302     vma = vma_lookup(current->mm, mmio_addr);
0303     if (!vma)
0304         goto out_unlock_mmap;
0305     if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
0306         goto out_unlock_mmap;
0307     ret = -EACCES;
0308     if (!(vma->vm_flags & VM_WRITE))
0309         goto out_unlock_mmap;
0310 
0311     ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl);
0312     if (ret)
0313         goto out_unlock_mmap;
0314 
0315     io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) |
0316             (mmio_addr & ~PAGE_MASK));
0317 
0318     if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) {
0319         ret = -EFAULT;
0320         goto out_unlock_pt;
0321     }
0322     ret = zpci_memcpy_fromio(buf, io_addr, length);
0323 
0324 out_unlock_pt:
0325     pte_unmap_unlock(ptep, ptl);
0326 out_unlock_mmap:
0327     mmap_read_unlock(current->mm);
0328 
0329     if (!ret && copy_to_user(user_buffer, buf, length))
0330         ret = -EFAULT;
0331 
0332     if (buf != local_buf)
0333         kfree(buf);
0334     return ret;
0335 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!