s390/pci/pci_insn.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * s390 specific pci instructions
0004  *
0005  * Copyright IBM Corp. 2013
0006  */
0007
0008 #include <linux/export.h>
0009 #include <linux/errno.h>
0010 #include <linux/delay.h>
0011 #include <linux/jump_label.h>
0012 #include <asm/asm-extable.h>
0013 #include <asm/facility.h>
0014 #include <asm/pci_insn.h>
0015 #include <asm/pci_debug.h>
0016 #include <asm/pci_io.h>
0017 #include <asm/processor.h>
0018
0019 #define ZPCI_INSN_BUSY_DELAY    1   /* 1 microsecond */
0020
0021 struct zpci_err_insn_data {
0022     u8 insn;
0023     u8 cc;
0024     u8 status;
0025     union {
0026         struct {
0027             u64 req;
0028             u64 offset;
0029         };
0030         struct {
0031             u64 addr;
0032             u64 len;
0033         };
0034     };
0035 } __packed;
0036
0037 static inline void zpci_err_insn_req(int lvl, u8 insn, u8 cc, u8 status,
0038                      u64 req, u64 offset)
0039 {
0040     struct zpci_err_insn_data data = {
0041         .insn = insn, .cc = cc, .status = status,
0042         .req = req, .offset = offset};
0043
0044     zpci_err_hex_level(lvl, &data, sizeof(data));
0045 }
0046
0047 static inline void zpci_err_insn_addr(int lvl, u8 insn, u8 cc, u8 status,
0048                       u64 addr, u64 len)
0049 {
0050     struct zpci_err_insn_data data = {
0051         .insn = insn, .cc = cc, .status = status,
0052         .addr = addr, .len = len};
0053
0054     zpci_err_hex_level(lvl, &data, sizeof(data));
0055 }
0056
0057 /* Modify PCI Function Controls */
0058 static inline u8 __mpcifc(u64 req, struct zpci_fib *fib, u8 *status)
0059 {
0060     u8 cc;
0061
0062     asm volatile (
0063         "   .insn   rxy,0xe300000000d0,%[req],%[fib]\n"
0064         "   ipm %[cc]\n"
0065         "   srl %[cc],28\n"
0066         : [cc] "=d" (cc), [req] "+d" (req), [fib] "+Q" (*fib)
0067         : : "cc");
0068     *status = req >> 24 & 0xff;
0069     return cc;
0070 }
0071
0072 u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status)
0073 {
0074     bool retried = false;
0075     u8 cc;
0076
0077     do {
0078         cc = __mpcifc(req, fib, status);
0079         if (cc == 2) {
0080             msleep(ZPCI_INSN_BUSY_DELAY);
0081             if (!retried) {
0082                 zpci_err_insn_req(1, 'M', cc, *status, req, 0);
0083                 retried = true;
0084             }
0085         }
0086     } while (cc == 2);
0087
0088     if (cc)
0089         zpci_err_insn_req(0, 'M', cc, *status, req, 0);
0090     else if (retried)
0091         zpci_err_insn_req(1, 'M', cc, *status, req, 0);
0092
0093     return cc;
0094 }
0095 EXPORT_SYMBOL_GPL(zpci_mod_fc);
0096
0097 /* Refresh PCI Translations */
0098 static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status)
0099 {
0100     union register_pair addr_range = {.even = addr, .odd = range};
0101     u8 cc;
0102
0103     asm volatile (
0104         "   .insn   rre,0xb9d30000,%[fn],%[addr_range]\n"
0105         "   ipm %[cc]\n"
0106         "   srl %[cc],28\n"
0107         : [cc] "=d" (cc), [fn] "+d" (fn)
0108         : [addr_range] "d" (addr_range.pair)
0109         : "cc");
0110     *status = fn >> 24 & 0xff;
0111     return cc;
0112 }
0113
0114 int zpci_refresh_trans(u64 fn, u64 addr, u64 range)
0115 {
0116     bool retried = false;
0117     u8 cc, status;
0118
0119     do {
0120         cc = __rpcit(fn, addr, range, &status);
0121         if (cc == 2) {
0122             udelay(ZPCI_INSN_BUSY_DELAY);
0123             if (!retried) {
0124                 zpci_err_insn_addr(1, 'R', cc, status, addr, range);
0125                 retried = true;
0126             }
0127         }
0128     } while (cc == 2);
0129
0130     if (cc)
0131         zpci_err_insn_addr(0, 'R', cc, status, addr, range);
0132     else if (retried)
0133         zpci_err_insn_addr(1, 'R', cc, status, addr, range);
0134
0135     if (cc == 1 && (status == 4 || status == 16))
0136         return -ENOMEM;
0137
0138     return (cc) ? -EIO : 0;
0139 }
0140
0141 /* Set Interruption Controls */
0142 int zpci_set_irq_ctrl(u16 ctl, u8 isc, union zpci_sic_iib *iib)
0143 {
0144     if (!test_facility(72))
0145         return -EIO;
0146
0147     asm volatile(
0148         ".insn  rsy,0xeb00000000d1,%[ctl],%[isc],%[iib]\n"
0149         : : [ctl] "d" (ctl), [isc] "d" (isc << 27), [iib] "Q" (*iib));
0150
0151     return 0;
0152 }
0153 EXPORT_SYMBOL_GPL(zpci_set_irq_ctrl);
0154
0155 /* PCI Load */
0156 static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status)
0157 {
0158     union register_pair req_off = {.even = req, .odd = offset};
0159     int cc = -ENXIO;
0160     u64 __data;
0161
0162     asm volatile (
0163         "   .insn   rre,0xb9d20000,%[data],%[req_off]\n"
0164         "0: ipm %[cc]\n"
0165         "   srl %[cc],28\n"
0166         "1:\n"
0167         EX_TABLE(0b, 1b)
0168         : [cc] "+d" (cc), [data] "=d" (__data),
0169           [req_off] "+&d" (req_off.pair) :: "cc");
0170     *status = req_off.even >> 24 & 0xff;
0171     *data = __data;
0172     return cc;
0173 }
0174
0175 static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status)
0176 {
0177     u64 __data;
0178     int cc;
0179
0180     cc = ____pcilg(&__data, req, offset, status);
0181     if (!cc)
0182         *data = __data;
0183
0184     return cc;
0185 }
0186
0187 int __zpci_load(u64 *data, u64 req, u64 offset)
0188 {
0189     bool retried = false;
0190     u8 status;
0191     int cc;
0192
0193     do {
0194         cc = __pcilg(data, req, offset, &status);
0195         if (cc == 2) {
0196             udelay(ZPCI_INSN_BUSY_DELAY);
0197             if (!retried) {
0198                 zpci_err_insn_req(1, 'l', cc, status, req, offset);
0199                 retried = true;
0200             }
0201         }
0202     } while (cc == 2);
0203
0204     if (cc)
0205         zpci_err_insn_req(0, 'l', cc, status, req, offset);
0206     else if (retried)
0207         zpci_err_insn_req(1, 'l', cc, status, req, offset);
0208
0209     return (cc > 0) ? -EIO : cc;
0210 }
0211 EXPORT_SYMBOL_GPL(__zpci_load);
0212
0213 static inline int zpci_load_fh(u64 *data, const volatile void __iomem *addr,
0214                    unsigned long len)
0215 {
0216     struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)];
0217     u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len);
0218
0219     return __zpci_load(data, req, ZPCI_OFFSET(addr));
0220 }
0221
0222 static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status)
0223 {
0224     union register_pair ioaddr_len = {.even = ioaddr, .odd = len};
0225     int cc = -ENXIO;
0226     u64 __data;
0227
0228     asm volatile (
0229         "       .insn   rre,0xb9d60000,%[data],%[ioaddr_len]\n"
0230         "0:     ipm     %[cc]\n"
0231         "       srl     %[cc],28\n"
0232         "1:\n"
0233         EX_TABLE(0b, 1b)
0234         : [cc] "+d" (cc), [data] "=d" (__data),
0235           [ioaddr_len] "+&d" (ioaddr_len.pair) :: "cc");
0236     *status = ioaddr_len.odd >> 24 & 0xff;
0237     *data = __data;
0238     return cc;
0239 }
0240
0241 int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len)
0242 {
0243     u8 status;
0244     int cc;
0245
0246     if (!static_branch_unlikely(&have_mio))
0247         return zpci_load_fh(data, addr, len);
0248
0249     cc = __pcilg_mio(data, (__force u64) addr, len, &status);
0250     if (cc)
0251         zpci_err_insn_addr(0, 'L', cc, status, (__force u64) addr, len);
0252
0253     return (cc > 0) ? -EIO : cc;
0254 }
0255 EXPORT_SYMBOL_GPL(zpci_load);
0256
0257 /* PCI Store */
0258 static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status)
0259 {
0260     union register_pair req_off = {.even = req, .odd = offset};
0261     int cc = -ENXIO;
0262
0263     asm volatile (
0264         "   .insn   rre,0xb9d00000,%[data],%[req_off]\n"
0265         "0: ipm %[cc]\n"
0266         "   srl %[cc],28\n"
0267         "1:\n"
0268         EX_TABLE(0b, 1b)
0269         : [cc] "+d" (cc), [req_off] "+&d" (req_off.pair)
0270         : [data] "d" (data)
0271         : "cc");
0272     *status = req_off.even >> 24 & 0xff;
0273     return cc;
0274 }
0275
0276 int __zpci_store(u64 data, u64 req, u64 offset)
0277 {
0278     bool retried = false;
0279     u8 status;
0280     int cc;
0281
0282     do {
0283         cc = __pcistg(data, req, offset, &status);
0284         if (cc == 2) {
0285             udelay(ZPCI_INSN_BUSY_DELAY);
0286             if (!retried) {
0287                 zpci_err_insn_req(1, 's', cc, status, req, offset);
0288                 retried = true;
0289             }
0290         }
0291     } while (cc == 2);
0292
0293     if (cc)
0294         zpci_err_insn_req(0, 's', cc, status, req, offset);
0295     else if (retried)
0296         zpci_err_insn_req(1, 's', cc, status, req, offset);
0297
0298     return (cc > 0) ? -EIO : cc;
0299 }
0300 EXPORT_SYMBOL_GPL(__zpci_store);
0301
0302 static inline int zpci_store_fh(const volatile void __iomem *addr, u64 data,
0303                 unsigned long len)
0304 {
0305     struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)];
0306     u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len);
0307
0308     return __zpci_store(data, req, ZPCI_OFFSET(addr));
0309 }
0310
0311 static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status)
0312 {
0313     union register_pair ioaddr_len = {.even = ioaddr, .odd = len};
0314     int cc = -ENXIO;
0315
0316     asm volatile (
0317         "       .insn   rre,0xb9d40000,%[data],%[ioaddr_len]\n"
0318         "0:     ipm     %[cc]\n"
0319         "       srl     %[cc],28\n"
0320         "1:\n"
0321         EX_TABLE(0b, 1b)
0322         : [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair)
0323         : [data] "d" (data)
0324         : "cc", "memory");
0325     *status = ioaddr_len.odd >> 24 & 0xff;
0326     return cc;
0327 }
0328
0329 int zpci_store(const volatile void __iomem *addr, u64 data, unsigned long len)
0330 {
0331     u8 status;
0332     int cc;
0333
0334     if (!static_branch_unlikely(&have_mio))
0335         return zpci_store_fh(addr, data, len);
0336
0337     cc = __pcistg_mio(data, (__force u64) addr, len, &status);
0338     if (cc)
0339         zpci_err_insn_addr(0, 'S', cc, status, (__force u64) addr, len);
0340
0341     return (cc > 0) ? -EIO : cc;
0342 }
0343 EXPORT_SYMBOL_GPL(zpci_store);
0344
0345 /* PCI Store Block */
0346 static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status)
0347 {
0348     int cc = -ENXIO;
0349
0350     asm volatile (
0351         "   .insn   rsy,0xeb00000000d0,%[req],%[offset],%[data]\n"
0352         "0: ipm %[cc]\n"
0353         "   srl %[cc],28\n"
0354         "1:\n"
0355         EX_TABLE(0b, 1b)
0356         : [cc] "+d" (cc), [req] "+d" (req)
0357         : [offset] "d" (offset), [data] "Q" (*data)
0358         : "cc");
0359     *status = req >> 24 & 0xff;
0360     return cc;
0361 }
0362
0363 int __zpci_store_block(const u64 *data, u64 req, u64 offset)
0364 {
0365     bool retried = false;
0366     u8 status;
0367     int cc;
0368
0369     do {
0370         cc = __pcistb(data, req, offset, &status);
0371         if (cc == 2) {
0372             udelay(ZPCI_INSN_BUSY_DELAY);
0373             if (!retried) {
0374                 zpci_err_insn_req(0, 'b', cc, status, req, offset);
0375                 retried = true;
0376             }
0377         }
0378     } while (cc == 2);
0379
0380     if (cc)
0381         zpci_err_insn_req(0, 'b', cc, status, req, offset);
0382     else if (retried)
0383         zpci_err_insn_req(1, 'b', cc, status, req, offset);
0384
0385     return (cc > 0) ? -EIO : cc;
0386 }
0387 EXPORT_SYMBOL_GPL(__zpci_store_block);
0388
0389 static inline int zpci_write_block_fh(volatile void __iomem *dst,
0390                       const void *src, unsigned long len)
0391 {
0392     struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(dst)];
0393     u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len);
0394     u64 offset = ZPCI_OFFSET(dst);
0395
0396     return __zpci_store_block(src, req, offset);
0397 }
0398
0399 static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status)
0400 {
0401     int cc = -ENXIO;
0402
0403     asm volatile (
0404         "       .insn   rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n"
0405         "0:     ipm     %[cc]\n"
0406         "       srl     %[cc],28\n"
0407         "1:\n"
0408         EX_TABLE(0b, 1b)
0409         : [cc] "+d" (cc), [len] "+d" (len)
0410         : [ioaddr] "d" (ioaddr), [data] "Q" (*data)
0411         : "cc");
0412     *status = len >> 24 & 0xff;
0413     return cc;
0414 }
0415
0416 int zpci_write_block(volatile void __iomem *dst,
0417              const void *src, unsigned long len)
0418 {
0419     u8 status;
0420     int cc;
0421
0422     if (!static_branch_unlikely(&have_mio))
0423         return zpci_write_block_fh(dst, src, len);
0424
0425     cc = __pcistb_mio(src, (__force u64) dst, len, &status);
0426     if (cc)
0427         zpci_err_insn_addr(0, 'B', cc, status, (__force u64) dst, len);
0428
0429     return (cc > 0) ? -EIO : cc;
0430 }
0431 EXPORT_SYMBOL_GPL(zpci_write_block);
0432
0433 static inline void __pciwb_mio(void)
0434 {
0435     asm volatile (".insn    rre,0xb9d50000,0,0\n");
0436 }
0437
0438 void zpci_barrier(void)
0439 {
0440     if (static_branch_likely(&have_mio))
0441         __pciwb_mio();
0442 }
0443 EXPORT_SYMBOL_GPL(zpci_barrier);