Back to home page

OSCL-LXR
 
 

     

0001 /*
0002  * Cavium ThunderX memory controller kernel module
0003  *
0004  * This file is subject to the terms and conditions of the GNU General Public
0005  * License.  See the file "COPYING" in the main directory of this archive
0006  * for more details.
0007  *
0008  * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
0009  *
0010  */
0011 
0012 #include <linux/module.h>
0013 #include <linux/pci.h>
0014 #include <linux/edac.h>
0015 #include <linux/interrupt.h>
0016 #include <linux/string.h>
0017 #include <linux/stop_machine.h>
0018 #include <linux/delay.h>
0019 #include <linux/sizes.h>
0020 #include <linux/atomic.h>
0021 #include <linux/bitfield.h>
0022 #include <linux/circ_buf.h>
0023 
0024 #include <asm/page.h>
0025 
0026 #include "edac_module.h"
0027 
0028 #define phys_to_pfn(phys)   (PFN_DOWN(phys))
0029 
0030 #define THUNDERX_NODE       GENMASK(45, 44)
0031 
0032 enum {
0033     ERR_CORRECTED   = 1,
0034     ERR_UNCORRECTED = 2,
0035     ERR_UNKNOWN = 3,
0036 };
0037 
0038 #define MAX_SYNDROME_REGS 4
0039 
0040 struct error_syndrome {
0041     u64 reg[MAX_SYNDROME_REGS];
0042 };
0043 
0044 struct error_descr {
0045     int type;
0046     u64 mask;
0047     char    *descr;
0048 };
0049 
0050 static void decode_register(char *str, size_t size,
0051                const struct error_descr *descr,
0052                const uint64_t reg)
0053 {
0054     int ret = 0;
0055 
0056     while (descr->type && descr->mask && descr->descr) {
0057         if (reg & descr->mask) {
0058             ret = snprintf(str, size, "\n\t%s, %s",
0059                        descr->type == ERR_CORRECTED ?
0060                      "Corrected" : "Uncorrected",
0061                        descr->descr);
0062             str += ret;
0063             size -= ret;
0064         }
0065         descr++;
0066     }
0067 }
0068 
0069 static unsigned long get_bits(unsigned long data, int pos, int width)
0070 {
0071     return (data >> pos) & ((1 << width) - 1);
0072 }
0073 
0074 #define L2C_CTL         0x87E080800000
0075 #define L2C_CTL_DISIDXALIAS BIT(0)
0076 
0077 #define PCI_DEVICE_ID_THUNDER_LMC 0xa022
0078 
0079 #define LMC_FADR        0x20
0080 #define LMC_FADR_FDIMM(x)   ((x >> 37) & 0x1)
0081 #define LMC_FADR_FBUNK(x)   ((x >> 36) & 0x1)
0082 #define LMC_FADR_FBANK(x)   ((x >> 32) & 0xf)
0083 #define LMC_FADR_FROW(x)    ((x >> 14) & 0xffff)
0084 #define LMC_FADR_FCOL(x)    ((x >> 0) & 0x1fff)
0085 
0086 #define LMC_NXM_FADR        0x28
0087 #define LMC_ECC_SYND        0x38
0088 
0089 #define LMC_ECC_PARITY_TEST 0x108
0090 
0091 #define LMC_INT_W1S     0x150
0092 
0093 #define LMC_INT_ENA_W1C     0x158
0094 #define LMC_INT_ENA_W1S     0x160
0095 
0096 #define LMC_CONFIG      0x188
0097 
0098 #define LMC_CONFIG_BG2      BIT(62)
0099 #define LMC_CONFIG_RANK_ENA BIT(42)
0100 #define LMC_CONFIG_PBANK_LSB(x) (((x) >> 5) & 0xF)
0101 #define LMC_CONFIG_ROW_LSB(x)   (((x) >> 2) & 0x7)
0102 
0103 #define LMC_CONTROL     0x190
0104 #define LMC_CONTROL_XOR_BANK    BIT(16)
0105 
0106 #define LMC_INT         0x1F0
0107 
0108 #define LMC_INT_DDR_ERR     BIT(11)
0109 #define LMC_INT_DED_ERR     (0xFUL << 5)
0110 #define LMC_INT_SEC_ERR         (0xFUL << 1)
0111 #define LMC_INT_NXM_WR_MASK BIT(0)
0112 
0113 #define LMC_DDR_PLL_CTL     0x258
0114 #define LMC_DDR_PLL_CTL_DDR4    BIT(29)
0115 
0116 #define LMC_FADR_SCRAMBLED  0x330
0117 
0118 #define LMC_INT_UE              (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \
0119                  LMC_INT_NXM_WR_MASK)
0120 
0121 #define LMC_INT_CE      (LMC_INT_SEC_ERR)
0122 
0123 static const struct error_descr lmc_errors[] = {
0124     {
0125         .type  = ERR_CORRECTED,
0126         .mask  = LMC_INT_SEC_ERR,
0127         .descr = "Single-bit ECC error",
0128     },
0129     {
0130         .type  = ERR_UNCORRECTED,
0131         .mask  = LMC_INT_DDR_ERR,
0132         .descr = "DDR chip error",
0133     },
0134     {
0135         .type  = ERR_UNCORRECTED,
0136         .mask  = LMC_INT_DED_ERR,
0137         .descr = "Double-bit ECC error",
0138     },
0139     {
0140         .type = ERR_UNCORRECTED,
0141         .mask = LMC_INT_NXM_WR_MASK,
0142         .descr = "Non-existent memory write",
0143     },
0144     {0, 0, NULL},
0145 };
0146 
0147 #define LMC_INT_EN_DDR_ERROR_ALERT_ENA  BIT(5)
0148 #define LMC_INT_EN_DLCRAM_DED_ERR   BIT(4)
0149 #define LMC_INT_EN_DLCRAM_SEC_ERR   BIT(3)
0150 #define LMC_INT_INTR_DED_ENA        BIT(2)
0151 #define LMC_INT_INTR_SEC_ENA        BIT(1)
0152 #define LMC_INT_INTR_NXM_WR_ENA     BIT(0)
0153 
0154 #define LMC_INT_ENA_ALL         GENMASK(5, 0)
0155 
0156 #define LMC_DDR_PLL_CTL     0x258
0157 #define LMC_DDR_PLL_CTL_DDR4    BIT(29)
0158 
0159 #define LMC_CONTROL     0x190
0160 #define LMC_CONTROL_RDIMM   BIT(0)
0161 
0162 #define LMC_SCRAM_FADR      0x330
0163 
0164 #define LMC_CHAR_MASK0      0x228
0165 #define LMC_CHAR_MASK2      0x238
0166 
0167 #define RING_ENTRIES    8
0168 
0169 struct debugfs_entry {
0170     const char *name;
0171     umode_t mode;
0172     const struct file_operations fops;
0173 };
0174 
0175 struct lmc_err_ctx {
0176     u64 reg_int;
0177     u64 reg_fadr;
0178     u64 reg_nxm_fadr;
0179     u64 reg_scram_fadr;
0180     u64 reg_ecc_synd;
0181 };
0182 
0183 struct thunderx_lmc {
0184     void __iomem *regs;
0185     struct pci_dev *pdev;
0186     struct msix_entry msix_ent;
0187 
0188     atomic_t ecc_int;
0189 
0190     u64 mask0;
0191     u64 mask2;
0192     u64 parity_test;
0193     u64 node;
0194 
0195     int xbits;
0196     int bank_width;
0197     int pbank_lsb;
0198     int dimm_lsb;
0199     int rank_lsb;
0200     int bank_lsb;
0201     int row_lsb;
0202     int col_hi_lsb;
0203 
0204     int xor_bank;
0205     int l2c_alias;
0206 
0207     struct page *mem;
0208 
0209     struct lmc_err_ctx err_ctx[RING_ENTRIES];
0210     unsigned long ring_head;
0211     unsigned long ring_tail;
0212 };
0213 
0214 #define ring_pos(pos, size) ((pos) & (size - 1))
0215 
0216 #define DEBUGFS_STRUCT(_name, _mode, _write, _read)             \
0217 static struct debugfs_entry debugfs_##_name = {                 \
0218     .name = __stringify(_name),                     \
0219     .mode = VERIFY_OCTAL_PERMISSIONS(_mode),                \
0220     .fops = {                               \
0221         .open = simple_open,                        \
0222         .write = _write,                        \
0223         .read  = _read,                         \
0224         .llseek = generic_file_llseek,                  \
0225     },                                  \
0226 }
0227 
0228 #define DEBUGFS_FIELD_ATTR(_type, _field)                   \
0229 static ssize_t thunderx_##_type##_##_field##_read(struct file *file,        \
0230                         char __user *data,          \
0231                         size_t count, loff_t *ppos)     \
0232 {                                       \
0233     struct thunderx_##_type *pdata = file->private_data;            \
0234     char buf[20];                               \
0235                                         \
0236     snprintf(buf, count, "0x%016llx", pdata->_field);           \
0237     return simple_read_from_buffer(data, count, ppos,           \
0238                        buf, sizeof(buf));           \
0239 }                                       \
0240                                         \
0241 static ssize_t thunderx_##_type##_##_field##_write(struct file *file,       \
0242                          const char __user *data,       \
0243                          size_t count, loff_t *ppos)    \
0244 {                                       \
0245     struct thunderx_##_type *pdata = file->private_data;            \
0246     int res;                                \
0247                                         \
0248     res = kstrtoull_from_user(data, count, 0, &pdata->_field);      \
0249                                         \
0250     return res ? res : count;                       \
0251 }                                       \
0252                                         \
0253 DEBUGFS_STRUCT(_field, 0600,                            \
0254            thunderx_##_type##_##_field##_write,             \
0255            thunderx_##_type##_##_field##_read)              \
0256 
0257 #define DEBUGFS_REG_ATTR(_type, _name, _reg)                    \
0258 static ssize_t thunderx_##_type##_##_name##_read(struct file *file,     \
0259                        char __user *data,           \
0260                        size_t count, loff_t *ppos)      \
0261 {                                       \
0262     struct thunderx_##_type *pdata = file->private_data;            \
0263     char buf[20];                               \
0264                                         \
0265     sprintf(buf, "0x%016llx", readq(pdata->regs + _reg));           \
0266     return simple_read_from_buffer(data, count, ppos,           \
0267                        buf, sizeof(buf));           \
0268 }                                       \
0269                                         \
0270 static ssize_t thunderx_##_type##_##_name##_write(struct file *file,        \
0271                         const char __user *data,        \
0272                         size_t count, loff_t *ppos)     \
0273 {                                       \
0274     struct thunderx_##_type *pdata = file->private_data;            \
0275     u64 val;                                \
0276     int res;                                \
0277                                         \
0278     res = kstrtoull_from_user(data, count, 0, &val);            \
0279                                         \
0280     if (!res) {                             \
0281         writeq(val, pdata->regs + _reg);                \
0282         res = count;                            \
0283     }                                   \
0284                                         \
0285     return res;                             \
0286 }                                       \
0287                                         \
0288 DEBUGFS_STRUCT(_name, 0600,                         \
0289            thunderx_##_type##_##_name##_write,              \
0290            thunderx_##_type##_##_name##_read)
0291 
0292 #define LMC_DEBUGFS_ENT(_field) DEBUGFS_FIELD_ATTR(lmc, _field)
0293 
0294 /*
0295  * To get an ECC error injected, the following steps are needed:
0296  * - Setup the ECC injection by writing the appropriate parameters:
0297  *  echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0
0298  *  echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2
0299  *  echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test
0300  * - Do the actual injection:
0301  *  echo 1 > /sys/kernel/debug/<device number>/inject_ecc
0302  */
0303 static ssize_t thunderx_lmc_inject_int_write(struct file *file,
0304                          const char __user *data,
0305                          size_t count, loff_t *ppos)
0306 {
0307     struct thunderx_lmc *lmc = file->private_data;
0308     u64 val;
0309     int res;
0310 
0311     res = kstrtoull_from_user(data, count, 0, &val);
0312 
0313     if (!res) {
0314         /* Trigger the interrupt */
0315         writeq(val, lmc->regs + LMC_INT_W1S);
0316         res = count;
0317     }
0318 
0319     return res;
0320 }
0321 
0322 static ssize_t thunderx_lmc_int_read(struct file *file,
0323                      char __user *data,
0324                      size_t count, loff_t *ppos)
0325 {
0326     struct thunderx_lmc *lmc = file->private_data;
0327     char buf[20];
0328     u64 lmc_int = readq(lmc->regs + LMC_INT);
0329 
0330     snprintf(buf, sizeof(buf), "0x%016llx", lmc_int);
0331     return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf));
0332 }
0333 
0334 #define TEST_PATTERN 0xa5
0335 
0336 static int inject_ecc_fn(void *arg)
0337 {
0338     struct thunderx_lmc *lmc = arg;
0339     uintptr_t addr, phys;
0340     unsigned int cline_size = cache_line_size();
0341     const unsigned int lines = PAGE_SIZE / cline_size;
0342     unsigned int i, cl_idx;
0343 
0344     addr = (uintptr_t)page_address(lmc->mem);
0345     phys = (uintptr_t)page_to_phys(lmc->mem);
0346 
0347     cl_idx = (phys & 0x7f) >> 4;
0348     lmc->parity_test &= ~(7ULL << 8);
0349     lmc->parity_test |= (cl_idx << 8);
0350 
0351     writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);
0352     writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);
0353     writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);
0354 
0355     readq(lmc->regs + LMC_CHAR_MASK0);
0356     readq(lmc->regs + LMC_CHAR_MASK2);
0357     readq(lmc->regs + LMC_ECC_PARITY_TEST);
0358 
0359     for (i = 0; i < lines; i++) {
0360         memset((void *)addr, TEST_PATTERN, cline_size);
0361         barrier();
0362 
0363         /*
0364          * Flush L1 cachelines to the PoC (L2).
0365          * This will cause cacheline eviction to the L2.
0366          */
0367         asm volatile("dc civac, %0\n"
0368                  "dsb sy\n"
0369                  : : "r"(addr + i * cline_size));
0370     }
0371 
0372     for (i = 0; i < lines; i++) {
0373         /*
0374          * Flush L2 cachelines to the DRAM.
0375          * This will cause cacheline eviction to the DRAM
0376          * and ECC corruption according to the masks set.
0377          */
0378         __asm__ volatile("sys #0,c11,C1,#2, %0\n"
0379                  : : "r"(phys + i * cline_size));
0380     }
0381 
0382     for (i = 0; i < lines; i++) {
0383         /*
0384          * Invalidate L2 cachelines.
0385          * The subsequent load will cause cacheline fetch
0386          * from the DRAM and an error interrupt
0387          */
0388         __asm__ volatile("sys #0,c11,C1,#1, %0"
0389                  : : "r"(phys + i * cline_size));
0390     }
0391 
0392     for (i = 0; i < lines; i++) {
0393         /*
0394          * Invalidate L1 cachelines.
0395          * The subsequent load will cause cacheline fetch
0396          * from the L2 and/or DRAM
0397          */
0398         asm volatile("dc ivac, %0\n"
0399                  "dsb sy\n"
0400                  : : "r"(addr + i * cline_size));
0401     }
0402 
0403     return 0;
0404 }
0405 
0406 static ssize_t thunderx_lmc_inject_ecc_write(struct file *file,
0407                          const char __user *data,
0408                          size_t count, loff_t *ppos)
0409 {
0410     struct thunderx_lmc *lmc = file->private_data;
0411     unsigned int cline_size = cache_line_size();
0412     u8 *tmp;
0413     void __iomem *addr;
0414     unsigned int offs, timeout = 100000;
0415 
0416     atomic_set(&lmc->ecc_int, 0);
0417 
0418     lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);
0419     if (!lmc->mem)
0420         return -ENOMEM;
0421 
0422     tmp = kmalloc(cline_size, GFP_KERNEL);
0423     if (!tmp) {
0424         __free_pages(lmc->mem, 0);
0425         return -ENOMEM;
0426     }
0427 
0428     addr = page_address(lmc->mem);
0429 
0430     while (!atomic_read(&lmc->ecc_int) && timeout--) {
0431         stop_machine(inject_ecc_fn, lmc, NULL);
0432 
0433         for (offs = 0; offs < PAGE_SIZE; offs += cline_size) {
0434             /*
0435              * Do a load from the previously rigged location
0436              * This should generate an error interrupt.
0437              */
0438             memcpy(tmp, addr + offs, cline_size);
0439             asm volatile("dsb ld\n");
0440         }
0441     }
0442 
0443     kfree(tmp);
0444     __free_pages(lmc->mem, 0);
0445 
0446     return count;
0447 }
0448 
0449 LMC_DEBUGFS_ENT(mask0);
0450 LMC_DEBUGFS_ENT(mask2);
0451 LMC_DEBUGFS_ENT(parity_test);
0452 
0453 DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL);
0454 DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
0455 DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
0456 
0457 static struct debugfs_entry *lmc_dfs_ents[] = {
0458     &debugfs_mask0,
0459     &debugfs_mask2,
0460     &debugfs_parity_test,
0461     &debugfs_inject_ecc,
0462     &debugfs_inject_int,
0463     &debugfs_int_w1c,
0464 };
0465 
0466 static int thunderx_create_debugfs_nodes(struct dentry *parent,
0467                       struct debugfs_entry *attrs[],
0468                       void *data,
0469                       size_t num)
0470 {
0471     int i;
0472     struct dentry *ent;
0473 
0474     if (!IS_ENABLED(CONFIG_EDAC_DEBUG))
0475         return 0;
0476 
0477     if (!parent)
0478         return -ENOENT;
0479 
0480     for (i = 0; i < num; i++) {
0481         ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,
0482                            parent, data, &attrs[i]->fops);
0483 
0484         if (!ent)
0485             break;
0486     }
0487 
0488     return i;
0489 }
0490 
0491 static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)
0492 {
0493     phys_addr_t addr = 0;
0494     int bank, xbits;
0495 
0496     addr |= lmc->node << 40;
0497     addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;
0498     addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;
0499     addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;
0500     addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;
0501 
0502     bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;
0503 
0504     if (lmc->xor_bank)
0505         bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);
0506 
0507     addr |= bank << lmc->bank_lsb;
0508 
0509     xbits = PCI_FUNC(lmc->pdev->devfn);
0510 
0511     if (lmc->l2c_alias)
0512         xbits ^= get_bits(addr, 20, lmc->xbits) ^
0513              get_bits(addr, 12, lmc->xbits);
0514 
0515     addr |= xbits << 7;
0516 
0517     return addr;
0518 }
0519 
0520 static unsigned int thunderx_get_num_lmcs(unsigned int node)
0521 {
0522     unsigned int number = 0;
0523     struct pci_dev *pdev = NULL;
0524 
0525     do {
0526         pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
0527                       PCI_DEVICE_ID_THUNDER_LMC,
0528                       pdev);
0529         if (pdev) {
0530 #ifdef CONFIG_NUMA
0531             if (pdev->dev.numa_node == node)
0532                 number++;
0533 #else
0534             number++;
0535 #endif
0536         }
0537     } while (pdev);
0538 
0539     return number;
0540 }
0541 
0542 #define LMC_MESSAGE_SIZE    120
0543 #define LMC_OTHER_SIZE      (50 * ARRAY_SIZE(lmc_errors))
0544 
0545 static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id)
0546 {
0547     struct mem_ctl_info *mci = dev_id;
0548     struct thunderx_lmc *lmc = mci->pvt_info;
0549 
0550     unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));
0551     struct lmc_err_ctx *ctx = &lmc->err_ctx[head];
0552 
0553     writeq(0, lmc->regs + LMC_CHAR_MASK0);
0554     writeq(0, lmc->regs + LMC_CHAR_MASK2);
0555     writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);
0556 
0557     ctx->reg_int = readq(lmc->regs + LMC_INT);
0558     ctx->reg_fadr = readq(lmc->regs + LMC_FADR);
0559     ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);
0560     ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);
0561     ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);
0562 
0563     lmc->ring_head++;
0564 
0565     atomic_set(&lmc->ecc_int, 1);
0566 
0567     /* Clear the interrupt */
0568     writeq(ctx->reg_int, lmc->regs + LMC_INT);
0569 
0570     return IRQ_WAKE_THREAD;
0571 }
0572 
0573 static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id)
0574 {
0575     struct mem_ctl_info *mci = dev_id;
0576     struct thunderx_lmc *lmc = mci->pvt_info;
0577     phys_addr_t phys_addr;
0578 
0579     unsigned long tail;
0580     struct lmc_err_ctx *ctx;
0581 
0582     irqreturn_t ret = IRQ_NONE;
0583 
0584     char *msg;
0585     char *other;
0586 
0587     msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL);
0588     other =  kmalloc(LMC_OTHER_SIZE, GFP_KERNEL);
0589 
0590     if (!msg || !other)
0591         goto err_free;
0592 
0593     while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,
0594         ARRAY_SIZE(lmc->err_ctx))) {
0595         tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));
0596 
0597         ctx = &lmc->err_ctx[tail];
0598 
0599         dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",
0600             ctx->reg_int);
0601         dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",
0602             ctx->reg_fadr);
0603         dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",
0604             ctx->reg_nxm_fadr);
0605         dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",
0606             ctx->reg_scram_fadr);
0607         dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",
0608             ctx->reg_ecc_synd);
0609 
0610         snprintf(msg, LMC_MESSAGE_SIZE,
0611              "DIMM %lld rank %lld bank %lld row %lld col %lld",
0612              LMC_FADR_FDIMM(ctx->reg_scram_fadr),
0613              LMC_FADR_FBUNK(ctx->reg_scram_fadr),
0614              LMC_FADR_FBANK(ctx->reg_scram_fadr),
0615              LMC_FADR_FROW(ctx->reg_scram_fadr),
0616              LMC_FADR_FCOL(ctx->reg_scram_fadr));
0617 
0618         decode_register(other, LMC_OTHER_SIZE, lmc_errors,
0619                 ctx->reg_int);
0620 
0621         phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);
0622 
0623         if (ctx->reg_int & LMC_INT_UE)
0624             edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
0625                          phys_to_pfn(phys_addr),
0626                          offset_in_page(phys_addr),
0627                          0, -1, -1, -1, msg, other);
0628         else if (ctx->reg_int & LMC_INT_CE)
0629             edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
0630                          phys_to_pfn(phys_addr),
0631                          offset_in_page(phys_addr),
0632                          0, -1, -1, -1, msg, other);
0633 
0634         lmc->ring_tail++;
0635     }
0636 
0637     ret = IRQ_HANDLED;
0638 
0639 err_free:
0640     kfree(msg);
0641     kfree(other);
0642 
0643     return ret;
0644 }
0645 
0646 static const struct pci_device_id thunderx_lmc_pci_tbl[] = {
0647     { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) },
0648     { 0, },
0649 };
0650 
0651 static inline int pci_dev_to_mc_idx(struct pci_dev *pdev)
0652 {
0653     int node = dev_to_node(&pdev->dev);
0654     int ret = PCI_FUNC(pdev->devfn);
0655 
0656     ret += max(node, 0) << 3;
0657 
0658     return ret;
0659 }
0660 
0661 static int thunderx_lmc_probe(struct pci_dev *pdev,
0662                 const struct pci_device_id *id)
0663 {
0664     struct thunderx_lmc *lmc;
0665     struct edac_mc_layer layer;
0666     struct mem_ctl_info *mci;
0667     u64 lmc_control, lmc_ddr_pll_ctl, lmc_config;
0668     int ret;
0669     u64 lmc_int;
0670     void *l2c_ioaddr;
0671 
0672     layer.type = EDAC_MC_LAYER_SLOT;
0673     layer.size = 2;
0674     layer.is_virt_csrow = false;
0675 
0676     ret = pcim_enable_device(pdev);
0677     if (ret) {
0678         dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
0679         return ret;
0680     }
0681 
0682     ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc");
0683     if (ret) {
0684         dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
0685         return ret;
0686     }
0687 
0688     mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer,
0689                 sizeof(struct thunderx_lmc));
0690     if (!mci)
0691         return -ENOMEM;
0692 
0693     mci->pdev = &pdev->dev;
0694     lmc = mci->pvt_info;
0695 
0696     pci_set_drvdata(pdev, mci);
0697 
0698     lmc->regs = pcim_iomap_table(pdev)[0];
0699 
0700     lmc_control = readq(lmc->regs + LMC_CONTROL);
0701     lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);
0702     lmc_config = readq(lmc->regs + LMC_CONFIG);
0703 
0704     if (lmc_control & LMC_CONTROL_RDIMM) {
0705         mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
0706                        lmc_ddr_pll_ctl) ?
0707                 MEM_RDDR4 : MEM_RDDR3;
0708     } else {
0709         mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
0710                        lmc_ddr_pll_ctl) ?
0711                 MEM_DDR4 : MEM_DDR3;
0712     }
0713 
0714     mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
0715     mci->edac_cap = EDAC_FLAG_SECDED;
0716 
0717     mci->mod_name = "thunderx-lmc";
0718     mci->ctl_name = "thunderx-lmc";
0719     mci->dev_name = dev_name(&pdev->dev);
0720     mci->scrub_mode = SCRUB_NONE;
0721 
0722     lmc->pdev = pdev;
0723     lmc->msix_ent.entry = 0;
0724 
0725     lmc->ring_head = 0;
0726     lmc->ring_tail = 0;
0727 
0728     ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);
0729     if (ret) {
0730         dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
0731         goto err_free;
0732     }
0733 
0734     ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,
0735                     thunderx_lmc_err_isr,
0736                     thunderx_lmc_threaded_isr, 0,
0737                     "[EDAC] ThunderX LMC", mci);
0738     if (ret) {
0739         dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret);
0740         goto err_free;
0741     }
0742 
0743     lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));
0744 
0745     lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;
0746     lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&
0747                FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3;
0748 
0749     lmc->pbank_lsb = (lmc_config >> 5) & 0xf;
0750     lmc->dimm_lsb  = 28 + lmc->pbank_lsb + lmc->xbits;
0751     lmc->rank_lsb = lmc->dimm_lsb;
0752     lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;
0753     lmc->bank_lsb = 7 + lmc->xbits;
0754     lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;
0755 
0756     lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;
0757 
0758     lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
0759 
0760     l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);
0761     if (!l2c_ioaddr) {
0762         dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
0763         ret = -ENOMEM;
0764         goto err_free;
0765     }
0766 
0767     lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);
0768 
0769     iounmap(l2c_ioaddr);
0770 
0771     ret = edac_mc_add_mc(mci);
0772     if (ret) {
0773         dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);
0774         goto err_free;
0775     }
0776 
0777     lmc_int = readq(lmc->regs + LMC_INT);
0778     writeq(lmc_int, lmc->regs + LMC_INT);
0779 
0780     writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);
0781 
0782     if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
0783         ret = thunderx_create_debugfs_nodes(mci->debugfs,
0784                             lmc_dfs_ents,
0785                             lmc,
0786                             ARRAY_SIZE(lmc_dfs_ents));
0787 
0788         if (ret != ARRAY_SIZE(lmc_dfs_ents)) {
0789             dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
0790                  ret, ret >= 0 ? " created" : "");
0791         }
0792     }
0793 
0794     return 0;
0795 
0796 err_free:
0797     pci_set_drvdata(pdev, NULL);
0798     edac_mc_free(mci);
0799 
0800     return ret;
0801 }
0802 
0803 static void thunderx_lmc_remove(struct pci_dev *pdev)
0804 {
0805     struct mem_ctl_info *mci = pci_get_drvdata(pdev);
0806     struct thunderx_lmc *lmc = mci->pvt_info;
0807 
0808     writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);
0809 
0810     edac_mc_del_mc(&pdev->dev);
0811     edac_mc_free(mci);
0812 }
0813 
0814 MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl);
0815 
0816 static struct pci_driver thunderx_lmc_driver = {
0817     .name     = "thunderx_lmc_edac",
0818     .probe    = thunderx_lmc_probe,
0819     .remove   = thunderx_lmc_remove,
0820     .id_table = thunderx_lmc_pci_tbl,
0821 };
0822 
0823 /*---------------------- OCX driver ---------------------------------*/
0824 
0825 #define PCI_DEVICE_ID_THUNDER_OCX 0xa013
0826 
0827 #define OCX_LINK_INTS       3
0828 #define OCX_INTS        (OCX_LINK_INTS + 1)
0829 #define OCX_RX_LANES        24
0830 #define OCX_RX_LANE_STATS   15
0831 
0832 #define OCX_COM_INT     0x100
0833 #define OCX_COM_INT_W1S     0x108
0834 #define OCX_COM_INT_ENA_W1S 0x110
0835 #define OCX_COM_INT_ENA_W1C 0x118
0836 
0837 #define OCX_COM_IO_BADID        BIT(54)
0838 #define OCX_COM_MEM_BADID       BIT(53)
0839 #define OCX_COM_COPR_BADID      BIT(52)
0840 #define OCX_COM_WIN_REQ_BADID       BIT(51)
0841 #define OCX_COM_WIN_REQ_TOUT        BIT(50)
0842 #define OCX_COM_RX_LANE         GENMASK(23, 0)
0843 
0844 #define OCX_COM_INT_CE          (OCX_COM_IO_BADID      | \
0845                      OCX_COM_MEM_BADID     | \
0846                      OCX_COM_COPR_BADID    | \
0847                      OCX_COM_WIN_REQ_BADID | \
0848                      OCX_COM_WIN_REQ_TOUT)
0849 
0850 static const struct error_descr ocx_com_errors[] = {
0851     {
0852         .type  = ERR_CORRECTED,
0853         .mask  = OCX_COM_IO_BADID,
0854         .descr = "Invalid IO transaction node ID",
0855     },
0856     {
0857         .type  = ERR_CORRECTED,
0858         .mask  = OCX_COM_MEM_BADID,
0859         .descr = "Invalid memory transaction node ID",
0860     },
0861     {
0862         .type  = ERR_CORRECTED,
0863         .mask  = OCX_COM_COPR_BADID,
0864         .descr = "Invalid coprocessor transaction node ID",
0865     },
0866     {
0867         .type  = ERR_CORRECTED,
0868         .mask  = OCX_COM_WIN_REQ_BADID,
0869         .descr = "Invalid SLI transaction node ID",
0870     },
0871     {
0872         .type  = ERR_CORRECTED,
0873         .mask  = OCX_COM_WIN_REQ_TOUT,
0874         .descr = "Window/core request timeout",
0875     },
0876     {0, 0, NULL},
0877 };
0878 
0879 #define OCX_COM_LINKX_INT(x)        (0x120 + (x) * 8)
0880 #define OCX_COM_LINKX_INT_W1S(x)    (0x140 + (x) * 8)
0881 #define OCX_COM_LINKX_INT_ENA_W1S(x)    (0x160 + (x) * 8)
0882 #define OCX_COM_LINKX_INT_ENA_W1C(x)    (0x180 + (x) * 8)
0883 
0884 #define OCX_COM_LINK_BAD_WORD           BIT(13)
0885 #define OCX_COM_LINK_ALIGN_FAIL         BIT(12)
0886 #define OCX_COM_LINK_ALIGN_DONE         BIT(11)
0887 #define OCX_COM_LINK_UP             BIT(10)
0888 #define OCX_COM_LINK_STOP           BIT(9)
0889 #define OCX_COM_LINK_BLK_ERR            BIT(8)
0890 #define OCX_COM_LINK_REINIT         BIT(7)
0891 #define OCX_COM_LINK_LNK_DATA           BIT(6)
0892 #define OCX_COM_LINK_RXFIFO_DBE         BIT(5)
0893 #define OCX_COM_LINK_RXFIFO_SBE         BIT(4)
0894 #define OCX_COM_LINK_TXFIFO_DBE         BIT(3)
0895 #define OCX_COM_LINK_TXFIFO_SBE         BIT(2)
0896 #define OCX_COM_LINK_REPLAY_DBE         BIT(1)
0897 #define OCX_COM_LINK_REPLAY_SBE         BIT(0)
0898 
0899 static const struct error_descr ocx_com_link_errors[] = {
0900     {
0901         .type  = ERR_CORRECTED,
0902         .mask  = OCX_COM_LINK_REPLAY_SBE,
0903         .descr = "Replay buffer single-bit error",
0904     },
0905     {
0906         .type  = ERR_CORRECTED,
0907         .mask  = OCX_COM_LINK_TXFIFO_SBE,
0908         .descr = "TX FIFO single-bit error",
0909     },
0910     {
0911         .type  = ERR_CORRECTED,
0912         .mask  = OCX_COM_LINK_RXFIFO_SBE,
0913         .descr = "RX FIFO single-bit error",
0914     },
0915     {
0916         .type  = ERR_CORRECTED,
0917         .mask  = OCX_COM_LINK_BLK_ERR,
0918         .descr = "Block code error",
0919     },
0920     {
0921         .type  = ERR_CORRECTED,
0922         .mask  = OCX_COM_LINK_ALIGN_FAIL,
0923         .descr = "Link alignment failure",
0924     },
0925     {
0926         .type  = ERR_CORRECTED,
0927         .mask  = OCX_COM_LINK_BAD_WORD,
0928         .descr = "Bad code word",
0929     },
0930     {
0931         .type  = ERR_UNCORRECTED,
0932         .mask  = OCX_COM_LINK_REPLAY_DBE,
0933         .descr = "Replay buffer double-bit error",
0934     },
0935     {
0936         .type  = ERR_UNCORRECTED,
0937         .mask  = OCX_COM_LINK_TXFIFO_DBE,
0938         .descr = "TX FIFO double-bit error",
0939     },
0940     {
0941         .type  = ERR_UNCORRECTED,
0942         .mask  = OCX_COM_LINK_RXFIFO_DBE,
0943         .descr = "RX FIFO double-bit error",
0944     },
0945     {
0946         .type  = ERR_UNCORRECTED,
0947         .mask  = OCX_COM_LINK_STOP,
0948         .descr = "Link stopped",
0949     },
0950     {0, 0, NULL},
0951 };
0952 
0953 #define OCX_COM_LINK_INT_UE       (OCX_COM_LINK_REPLAY_DBE | \
0954                    OCX_COM_LINK_TXFIFO_DBE | \
0955                    OCX_COM_LINK_RXFIFO_DBE | \
0956                    OCX_COM_LINK_STOP)
0957 
0958 #define OCX_COM_LINK_INT_CE       (OCX_COM_LINK_REPLAY_SBE | \
0959                    OCX_COM_LINK_TXFIFO_SBE | \
0960                    OCX_COM_LINK_RXFIFO_SBE | \
0961                    OCX_COM_LINK_BLK_ERR    | \
0962                    OCX_COM_LINK_ALIGN_FAIL | \
0963                    OCX_COM_LINK_BAD_WORD)
0964 
0965 #define OCX_LNE_INT(x)          (0x8018 + (x) * 0x100)
0966 #define OCX_LNE_INT_EN(x)       (0x8020 + (x) * 0x100)
0967 #define OCX_LNE_BAD_CNT(x)      (0x8028 + (x) * 0x100)
0968 #define OCX_LNE_CFG(x)          (0x8000 + (x) * 0x100)
0969 #define OCX_LNE_STAT(x, y)      (0x8040 + (x) * 0x100 + (y) * 8)
0970 
0971 #define OCX_LNE_CFG_RX_BDRY_LOCK_DIS        BIT(8)
0972 #define OCX_LNE_CFG_RX_STAT_WRAP_DIS        BIT(2)
0973 #define OCX_LNE_CFG_RX_STAT_RDCLR       BIT(1)
0974 #define OCX_LNE_CFG_RX_STAT_ENA         BIT(0)
0975 
0976 
0977 #define OCX_LANE_BAD_64B67B         BIT(8)
0978 #define OCX_LANE_DSKEW_FIFO_OVFL        BIT(5)
0979 #define OCX_LANE_SCRM_SYNC_LOSS         BIT(4)
0980 #define OCX_LANE_UKWN_CNTL_WORD         BIT(3)
0981 #define OCX_LANE_CRC32_ERR          BIT(2)
0982 #define OCX_LANE_BDRY_SYNC_LOSS         BIT(1)
0983 #define OCX_LANE_SERDES_LOCK_LOSS       BIT(0)
0984 
0985 #define OCX_COM_LANE_INT_UE       (0)
0986 #define OCX_COM_LANE_INT_CE       (OCX_LANE_SERDES_LOCK_LOSS | \
0987                    OCX_LANE_BDRY_SYNC_LOSS   | \
0988                    OCX_LANE_CRC32_ERR        | \
0989                    OCX_LANE_UKWN_CNTL_WORD   | \
0990                    OCX_LANE_SCRM_SYNC_LOSS   | \
0991                    OCX_LANE_DSKEW_FIFO_OVFL  | \
0992                    OCX_LANE_BAD_64B67B)
0993 
0994 static const struct error_descr ocx_lane_errors[] = {
0995     {
0996         .type  = ERR_CORRECTED,
0997         .mask  = OCX_LANE_SERDES_LOCK_LOSS,
0998         .descr = "RX SerDes lock lost",
0999     },
1000     {
1001         .type  = ERR_CORRECTED,
1002         .mask  = OCX_LANE_BDRY_SYNC_LOSS,
1003         .descr = "RX word boundary lost",
1004     },
1005     {
1006         .type  = ERR_CORRECTED,
1007         .mask  = OCX_LANE_CRC32_ERR,
1008         .descr = "CRC32 error",
1009     },
1010     {
1011         .type  = ERR_CORRECTED,
1012         .mask  = OCX_LANE_UKWN_CNTL_WORD,
1013         .descr = "Unknown control word",
1014     },
1015     {
1016         .type  = ERR_CORRECTED,
1017         .mask  = OCX_LANE_SCRM_SYNC_LOSS,
1018         .descr = "Scrambler synchronization lost",
1019     },
1020     {
1021         .type  = ERR_CORRECTED,
1022         .mask  = OCX_LANE_DSKEW_FIFO_OVFL,
1023         .descr = "RX deskew FIFO overflow",
1024     },
1025     {
1026         .type  = ERR_CORRECTED,
1027         .mask  = OCX_LANE_BAD_64B67B,
1028         .descr = "Bad 64B/67B codeword",
1029     },
1030     {0, 0, NULL},
1031 };
1032 
1033 #define OCX_LNE_INT_ENA_ALL     (GENMASK(9, 8) | GENMASK(6, 0))
1034 #define OCX_COM_INT_ENA_ALL     (GENMASK(54, 50) | GENMASK(23, 0))
1035 #define OCX_COM_LINKX_INT_ENA_ALL   (GENMASK(13, 12) | \
1036                      GENMASK(9, 7) | GENMASK(5, 0))
1037 
1038 #define OCX_TLKX_ECC_CTL(x)     (0x10018 + (x) * 0x2000)
1039 #define OCX_RLKX_ECC_CTL(x)     (0x18018 + (x) * 0x2000)
1040 
1041 struct ocx_com_err_ctx {
1042     u64 reg_com_int;
1043     u64 reg_lane_int[OCX_RX_LANES];
1044     u64 reg_lane_stat11[OCX_RX_LANES];
1045 };
1046 
1047 struct ocx_link_err_ctx {
1048     u64 reg_com_link_int;
1049     int link;
1050 };
1051 
1052 struct thunderx_ocx {
1053     void __iomem *regs;
1054     int com_link;
1055     struct pci_dev *pdev;
1056     struct edac_device_ctl_info *edac_dev;
1057 
1058     struct dentry *debugfs;
1059     struct msix_entry msix_ent[OCX_INTS];
1060 
1061     struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES];
1062     struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES];
1063 
1064     unsigned long com_ring_head;
1065     unsigned long com_ring_tail;
1066 
1067     unsigned long link_ring_head;
1068     unsigned long link_ring_tail;
1069 };
1070 
1071 #define OCX_MESSAGE_SIZE    SZ_1K
1072 #define OCX_OTHER_SIZE      (50 * ARRAY_SIZE(ocx_com_link_errors))
1073 
1074 /* This handler is threaded */
1075 static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id)
1076 {
1077     struct msix_entry *msix = irq_id;
1078     struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1079                         msix_ent[msix->entry]);
1080 
1081     int lane;
1082     unsigned long head = ring_pos(ocx->com_ring_head,
1083                       ARRAY_SIZE(ocx->com_err_ctx));
1084     struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];
1085 
1086     ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);
1087 
1088     for (lane = 0; lane < OCX_RX_LANES; lane++) {
1089         ctx->reg_lane_int[lane] =
1090             readq(ocx->regs + OCX_LNE_INT(lane));
1091         ctx->reg_lane_stat11[lane] =
1092             readq(ocx->regs + OCX_LNE_STAT(lane, 11));
1093 
1094         writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));
1095     }
1096 
1097     writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);
1098 
1099     ocx->com_ring_head++;
1100 
1101     return IRQ_WAKE_THREAD;
1102 }
1103 
1104 static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id)
1105 {
1106     struct msix_entry *msix = irq_id;
1107     struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1108                         msix_ent[msix->entry]);
1109 
1110     irqreturn_t ret = IRQ_NONE;
1111 
1112     unsigned long tail;
1113     struct ocx_com_err_ctx *ctx;
1114     int lane;
1115     char *msg;
1116     char *other;
1117 
1118     msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1119     other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1120 
1121     if (!msg || !other)
1122         goto err_free;
1123 
1124     while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,
1125             ARRAY_SIZE(ocx->com_err_ctx))) {
1126         tail = ring_pos(ocx->com_ring_tail,
1127                 ARRAY_SIZE(ocx->com_err_ctx));
1128         ctx = &ocx->com_err_ctx[tail];
1129 
1130         snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx",
1131             ocx->edac_dev->ctl_name, ctx->reg_com_int);
1132 
1133         decode_register(other, OCX_OTHER_SIZE,
1134                 ocx_com_errors, ctx->reg_com_int);
1135 
1136         strncat(msg, other, OCX_MESSAGE_SIZE);
1137 
1138         for (lane = 0; lane < OCX_RX_LANES; lane++)
1139             if (ctx->reg_com_int & BIT(lane)) {
1140                 snprintf(other, OCX_OTHER_SIZE,
1141                      "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx",
1142                      lane, ctx->reg_lane_int[lane],
1143                      lane, ctx->reg_lane_stat11[lane]);
1144 
1145                 strncat(msg, other, OCX_MESSAGE_SIZE);
1146 
1147                 decode_register(other, OCX_OTHER_SIZE,
1148                         ocx_lane_errors,
1149                         ctx->reg_lane_int[lane]);
1150                 strncat(msg, other, OCX_MESSAGE_SIZE);
1151             }
1152 
1153         if (ctx->reg_com_int & OCX_COM_INT_CE)
1154             edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1155 
1156         ocx->com_ring_tail++;
1157     }
1158 
1159     ret = IRQ_HANDLED;
1160 
1161 err_free:
1162     kfree(other);
1163     kfree(msg);
1164 
1165     return ret;
1166 }
1167 
1168 static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id)
1169 {
1170     struct msix_entry *msix = irq_id;
1171     struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1172                         msix_ent[msix->entry]);
1173     unsigned long head = ring_pos(ocx->link_ring_head,
1174                       ARRAY_SIZE(ocx->link_err_ctx));
1175     struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];
1176 
1177     ctx->link = msix->entry;
1178     ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1179 
1180     writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1181 
1182     ocx->link_ring_head++;
1183 
1184     return IRQ_WAKE_THREAD;
1185 }
1186 
1187 static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id)
1188 {
1189     struct msix_entry *msix = irq_id;
1190     struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1191                         msix_ent[msix->entry]);
1192     irqreturn_t ret = IRQ_NONE;
1193     unsigned long tail;
1194     struct ocx_link_err_ctx *ctx;
1195 
1196     char *msg;
1197     char *other;
1198 
1199     msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1200     other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1201 
1202     if (!msg || !other)
1203         goto err_free;
1204 
1205     while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,
1206             ARRAY_SIZE(ocx->link_err_ctx))) {
1207         tail = ring_pos(ocx->link_ring_head,
1208                 ARRAY_SIZE(ocx->link_err_ctx));
1209 
1210         ctx = &ocx->link_err_ctx[tail];
1211 
1212         snprintf(msg, OCX_MESSAGE_SIZE,
1213              "%s: OCX_COM_LINK_INT[%d]: %016llx",
1214              ocx->edac_dev->ctl_name,
1215              ctx->link, ctx->reg_com_link_int);
1216 
1217         decode_register(other, OCX_OTHER_SIZE,
1218                 ocx_com_link_errors, ctx->reg_com_link_int);
1219 
1220         strncat(msg, other, OCX_MESSAGE_SIZE);
1221 
1222         if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)
1223             edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
1224         else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)
1225             edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1226 
1227         ocx->link_ring_tail++;
1228     }
1229 
1230     ret = IRQ_HANDLED;
1231 err_free:
1232     kfree(other);
1233     kfree(msg);
1234 
1235     return ret;
1236 }
1237 
1238 #define OCX_DEBUGFS_ATTR(_name, _reg)   DEBUGFS_REG_ATTR(ocx, _name, _reg)
1239 
1240 OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0));
1241 OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1));
1242 OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2));
1243 
1244 OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0));
1245 OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1));
1246 OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2));
1247 
1248 OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0));
1249 OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1));
1250 OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2));
1251 
1252 OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0));
1253 OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1));
1254 OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2));
1255 OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3));
1256 OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4));
1257 OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5));
1258 OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6));
1259 OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7));
1260 
1261 OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8));
1262 OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9));
1263 OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10));
1264 OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11));
1265 OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12));
1266 OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13));
1267 OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14));
1268 OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15));
1269 
1270 OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16));
1271 OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17));
1272 OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18));
1273 OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19));
1274 OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20));
1275 OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21));
1276 OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22));
1277 OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23));
1278 
1279 OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S);
1280 
1281 static struct debugfs_entry *ocx_dfs_ents[] = {
1282     &debugfs_tlk0_ecc_ctl,
1283     &debugfs_tlk1_ecc_ctl,
1284     &debugfs_tlk2_ecc_ctl,
1285 
1286     &debugfs_rlk0_ecc_ctl,
1287     &debugfs_rlk1_ecc_ctl,
1288     &debugfs_rlk2_ecc_ctl,
1289 
1290     &debugfs_com_link0_int,
1291     &debugfs_com_link1_int,
1292     &debugfs_com_link2_int,
1293 
1294     &debugfs_lne00_badcnt,
1295     &debugfs_lne01_badcnt,
1296     &debugfs_lne02_badcnt,
1297     &debugfs_lne03_badcnt,
1298     &debugfs_lne04_badcnt,
1299     &debugfs_lne05_badcnt,
1300     &debugfs_lne06_badcnt,
1301     &debugfs_lne07_badcnt,
1302     &debugfs_lne08_badcnt,
1303     &debugfs_lne09_badcnt,
1304     &debugfs_lne10_badcnt,
1305     &debugfs_lne11_badcnt,
1306     &debugfs_lne12_badcnt,
1307     &debugfs_lne13_badcnt,
1308     &debugfs_lne14_badcnt,
1309     &debugfs_lne15_badcnt,
1310     &debugfs_lne16_badcnt,
1311     &debugfs_lne17_badcnt,
1312     &debugfs_lne18_badcnt,
1313     &debugfs_lne19_badcnt,
1314     &debugfs_lne20_badcnt,
1315     &debugfs_lne21_badcnt,
1316     &debugfs_lne22_badcnt,
1317     &debugfs_lne23_badcnt,
1318 
1319     &debugfs_com_int,
1320 };
1321 
1322 static const struct pci_device_id thunderx_ocx_pci_tbl[] = {
1323     { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) },
1324     { 0, },
1325 };
1326 
1327 static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx)
1328 {
1329     int lane, stat, cfg;
1330 
1331     for (lane = 0; lane < OCX_RX_LANES; lane++) {
1332         cfg = readq(ocx->regs + OCX_LNE_CFG(lane));
1333         cfg |= OCX_LNE_CFG_RX_STAT_RDCLR;
1334         cfg &= ~OCX_LNE_CFG_RX_STAT_ENA;
1335         writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));
1336 
1337         for (stat = 0; stat < OCX_RX_LANE_STATS; stat++)
1338             readq(ocx->regs + OCX_LNE_STAT(lane, stat));
1339     }
1340 }
1341 
1342 static int thunderx_ocx_probe(struct pci_dev *pdev,
1343                   const struct pci_device_id *id)
1344 {
1345     struct thunderx_ocx *ocx;
1346     struct edac_device_ctl_info *edac_dev;
1347     char name[32];
1348     int idx;
1349     int i;
1350     int ret;
1351     u64 reg;
1352 
1353     ret = pcim_enable_device(pdev);
1354     if (ret) {
1355         dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1356         return ret;
1357     }
1358 
1359     ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx");
1360     if (ret) {
1361         dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1362         return ret;
1363     }
1364 
1365     idx = edac_device_alloc_index();
1366     snprintf(name, sizeof(name), "OCX%d", idx);
1367     edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx),
1368                           name, 1, "CCPI", 1,
1369                           0, NULL, 0, idx);
1370     if (!edac_dev) {
1371         dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
1372         return -ENOMEM;
1373     }
1374     ocx = edac_dev->pvt_info;
1375     ocx->edac_dev = edac_dev;
1376     ocx->com_ring_head = 0;
1377     ocx->com_ring_tail = 0;
1378     ocx->link_ring_head = 0;
1379     ocx->link_ring_tail = 0;
1380 
1381     ocx->regs = pcim_iomap_table(pdev)[0];
1382     if (!ocx->regs) {
1383         dev_err(&pdev->dev, "Cannot map PCI resources\n");
1384         ret = -ENODEV;
1385         goto err_free;
1386     }
1387 
1388     ocx->pdev = pdev;
1389 
1390     for (i = 0; i < OCX_INTS; i++) {
1391         ocx->msix_ent[i].entry = i;
1392         ocx->msix_ent[i].vector = 0;
1393     }
1394 
1395     ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);
1396     if (ret) {
1397         dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
1398         goto err_free;
1399     }
1400 
1401     for (i = 0; i < OCX_INTS; i++) {
1402         ret = devm_request_threaded_irq(&pdev->dev,
1403                         ocx->msix_ent[i].vector,
1404                         (i == 3) ?
1405                          thunderx_ocx_com_isr :
1406                          thunderx_ocx_lnk_isr,
1407                         (i == 3) ?
1408                          thunderx_ocx_com_threaded_isr :
1409                          thunderx_ocx_lnk_threaded_isr,
1410                         0, "[EDAC] ThunderX OCX",
1411                         &ocx->msix_ent[i]);
1412         if (ret)
1413             goto err_free;
1414     }
1415 
1416     edac_dev->dev = &pdev->dev;
1417     edac_dev->dev_name = dev_name(&pdev->dev);
1418     edac_dev->mod_name = "thunderx-ocx";
1419     edac_dev->ctl_name = "thunderx-ocx";
1420 
1421     ret = edac_device_add_device(edac_dev);
1422     if (ret) {
1423         dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
1424         goto err_free;
1425     }
1426 
1427     if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
1428         ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
1429 
1430         ret = thunderx_create_debugfs_nodes(ocx->debugfs,
1431                             ocx_dfs_ents,
1432                             ocx,
1433                             ARRAY_SIZE(ocx_dfs_ents));
1434         if (ret != ARRAY_SIZE(ocx_dfs_ents)) {
1435             dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
1436                  ret, ret >= 0 ? " created" : "");
1437         }
1438     }
1439 
1440     pci_set_drvdata(pdev, edac_dev);
1441 
1442     thunderx_ocx_clearstats(ocx);
1443 
1444     for (i = 0; i < OCX_RX_LANES; i++) {
1445         writeq(OCX_LNE_INT_ENA_ALL,
1446                ocx->regs + OCX_LNE_INT_EN(i));
1447 
1448         reg = readq(ocx->regs + OCX_LNE_INT(i));
1449         writeq(reg, ocx->regs + OCX_LNE_INT(i));
1450 
1451     }
1452 
1453     for (i = 0; i < OCX_LINK_INTS; i++) {
1454         reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));
1455         writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));
1456 
1457         writeq(OCX_COM_LINKX_INT_ENA_ALL,
1458                ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));
1459     }
1460 
1461     reg = readq(ocx->regs + OCX_COM_INT);
1462     writeq(reg, ocx->regs + OCX_COM_INT);
1463 
1464     writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);
1465 
1466     return 0;
1467 err_free:
1468     edac_device_free_ctl_info(edac_dev);
1469 
1470     return ret;
1471 }
1472 
1473 static void thunderx_ocx_remove(struct pci_dev *pdev)
1474 {
1475     struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
1476     struct thunderx_ocx *ocx = edac_dev->pvt_info;
1477     int i;
1478 
1479     writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);
1480 
1481     for (i = 0; i < OCX_INTS; i++) {
1482         writeq(OCX_COM_LINKX_INT_ENA_ALL,
1483                ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));
1484     }
1485 
1486     edac_debugfs_remove_recursive(ocx->debugfs);
1487 
1488     edac_device_del_device(&pdev->dev);
1489     edac_device_free_ctl_info(edac_dev);
1490 }
1491 
1492 MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl);
1493 
1494 static struct pci_driver thunderx_ocx_driver = {
1495     .name     = "thunderx_ocx_edac",
1496     .probe    = thunderx_ocx_probe,
1497     .remove   = thunderx_ocx_remove,
1498     .id_table = thunderx_ocx_pci_tbl,
1499 };
1500 
1501 /*---------------------- L2C driver ---------------------------------*/
1502 
1503 #define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e
1504 #define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f
1505 #define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030
1506 
1507 #define L2C_TAD_INT_W1C     0x40000
1508 #define L2C_TAD_INT_W1S     0x40008
1509 
1510 #define L2C_TAD_INT_ENA_W1C 0x40020
1511 #define L2C_TAD_INT_ENA_W1S 0x40028
1512 
1513 
1514 #define L2C_TAD_INT_L2DDBE   BIT(1)
1515 #define L2C_TAD_INT_SBFSBE   BIT(2)
1516 #define L2C_TAD_INT_SBFDBE   BIT(3)
1517 #define L2C_TAD_INT_FBFSBE   BIT(4)
1518 #define L2C_TAD_INT_FBFDBE   BIT(5)
1519 #define L2C_TAD_INT_TAGDBE   BIT(9)
1520 #define L2C_TAD_INT_RDDISLMC     BIT(15)
1521 #define L2C_TAD_INT_WRDISLMC     BIT(16)
1522 #define L2C_TAD_INT_LFBTO    BIT(17)
1523 #define L2C_TAD_INT_GSYNCTO  BIT(18)
1524 #define L2C_TAD_INT_RTGSBE   BIT(32)
1525 #define L2C_TAD_INT_RTGDBE   BIT(33)
1526 #define L2C_TAD_INT_RDDISOCI     BIT(34)
1527 #define L2C_TAD_INT_WRDISOCI     BIT(35)
1528 
1529 #define L2C_TAD_INT_ECC     (L2C_TAD_INT_L2DDBE | \
1530                  L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \
1531                  L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE)
1532 
1533 #define L2C_TAD_INT_CE          (L2C_TAD_INT_SBFSBE | \
1534                  L2C_TAD_INT_FBFSBE)
1535 
1536 #define L2C_TAD_INT_UE          (L2C_TAD_INT_L2DDBE | \
1537                  L2C_TAD_INT_SBFDBE | \
1538                  L2C_TAD_INT_FBFDBE | \
1539                  L2C_TAD_INT_TAGDBE | \
1540                  L2C_TAD_INT_RTGDBE | \
1541                  L2C_TAD_INT_WRDISOCI | \
1542                  L2C_TAD_INT_RDDISOCI | \
1543                  L2C_TAD_INT_WRDISLMC | \
1544                  L2C_TAD_INT_RDDISLMC | \
1545                  L2C_TAD_INT_LFBTO    | \
1546                  L2C_TAD_INT_GSYNCTO)
1547 
1548 static const struct error_descr l2_tad_errors[] = {
1549     {
1550         .type  = ERR_CORRECTED,
1551         .mask  = L2C_TAD_INT_SBFSBE,
1552         .descr = "SBF single-bit error",
1553     },
1554     {
1555         .type  = ERR_CORRECTED,
1556         .mask  = L2C_TAD_INT_FBFSBE,
1557         .descr = "FBF single-bit error",
1558     },
1559     {
1560         .type  = ERR_UNCORRECTED,
1561         .mask  = L2C_TAD_INT_L2DDBE,
1562         .descr = "L2D double-bit error",
1563     },
1564     {
1565         .type  = ERR_UNCORRECTED,
1566         .mask  = L2C_TAD_INT_SBFDBE,
1567         .descr = "SBF double-bit error",
1568     },
1569     {
1570         .type  = ERR_UNCORRECTED,
1571         .mask  = L2C_TAD_INT_FBFDBE,
1572         .descr = "FBF double-bit error",
1573     },
1574     {
1575         .type  = ERR_UNCORRECTED,
1576         .mask  = L2C_TAD_INT_TAGDBE,
1577         .descr = "TAG double-bit error",
1578     },
1579     {
1580         .type  = ERR_UNCORRECTED,
1581         .mask  = L2C_TAD_INT_RTGDBE,
1582         .descr = "RTG double-bit error",
1583     },
1584     {
1585         .type  = ERR_UNCORRECTED,
1586         .mask  = L2C_TAD_INT_WRDISOCI,
1587         .descr = "Write to a disabled CCPI",
1588     },
1589     {
1590         .type  = ERR_UNCORRECTED,
1591         .mask  = L2C_TAD_INT_RDDISOCI,
1592         .descr = "Read from a disabled CCPI",
1593     },
1594     {
1595         .type  = ERR_UNCORRECTED,
1596         .mask  = L2C_TAD_INT_WRDISLMC,
1597         .descr = "Write to a disabled LMC",
1598     },
1599     {
1600         .type  = ERR_UNCORRECTED,
1601         .mask  = L2C_TAD_INT_RDDISLMC,
1602         .descr = "Read from a disabled LMC",
1603     },
1604     {
1605         .type  = ERR_UNCORRECTED,
1606         .mask  = L2C_TAD_INT_LFBTO,
1607         .descr = "LFB entry timeout",
1608     },
1609     {
1610         .type  = ERR_UNCORRECTED,
1611         .mask  = L2C_TAD_INT_GSYNCTO,
1612         .descr = "Global sync CCPI timeout",
1613     },
1614     {0, 0, NULL},
1615 };
1616 
1617 #define L2C_TAD_INT_TAG     (L2C_TAD_INT_TAGDBE)
1618 
1619 #define L2C_TAD_INT_RTG     (L2C_TAD_INT_RTGDBE)
1620 
1621 #define L2C_TAD_INT_DISLMC  (L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC)
1622 
1623 #define L2C_TAD_INT_DISOCI  (L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI)
1624 
1625 #define L2C_TAD_INT_ENA_ALL (L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \
1626                  L2C_TAD_INT_RTG | \
1627                  L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \
1628                  L2C_TAD_INT_LFBTO)
1629 
1630 #define L2C_TAD_TIMETWO     0x50000
1631 #define L2C_TAD_TIMEOUT     0x50100
1632 #define L2C_TAD_ERR     0x60000
1633 #define L2C_TAD_TQD_ERR     0x60100
1634 #define L2C_TAD_TTG_ERR     0x60200
1635 
1636 
1637 #define L2C_CBC_INT_W1C     0x60000
1638 
1639 #define L2C_CBC_INT_RSDSBE   BIT(0)
1640 #define L2C_CBC_INT_RSDDBE   BIT(1)
1641 
1642 #define L2C_CBC_INT_RSD      (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE)
1643 
1644 #define L2C_CBC_INT_MIBSBE   BIT(4)
1645 #define L2C_CBC_INT_MIBDBE   BIT(5)
1646 
1647 #define L2C_CBC_INT_MIB      (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE)
1648 
1649 #define L2C_CBC_INT_IORDDISOCI   BIT(6)
1650 #define L2C_CBC_INT_IOWRDISOCI   BIT(7)
1651 
1652 #define L2C_CBC_INT_IODISOCI     (L2C_CBC_INT_IORDDISOCI | \
1653                   L2C_CBC_INT_IOWRDISOCI)
1654 
1655 #define L2C_CBC_INT_CE       (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE)
1656 #define L2C_CBC_INT_UE       (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE)
1657 
1658 
1659 static const struct error_descr l2_cbc_errors[] = {
1660     {
1661         .type  = ERR_CORRECTED,
1662         .mask  = L2C_CBC_INT_RSDSBE,
1663         .descr = "RSD single-bit error",
1664     },
1665     {
1666         .type  = ERR_CORRECTED,
1667         .mask  = L2C_CBC_INT_MIBSBE,
1668         .descr = "MIB single-bit error",
1669     },
1670     {
1671         .type  = ERR_UNCORRECTED,
1672         .mask  = L2C_CBC_INT_RSDDBE,
1673         .descr = "RSD double-bit error",
1674     },
1675     {
1676         .type  = ERR_UNCORRECTED,
1677         .mask  = L2C_CBC_INT_MIBDBE,
1678         .descr = "MIB double-bit error",
1679     },
1680     {
1681         .type  = ERR_UNCORRECTED,
1682         .mask  = L2C_CBC_INT_IORDDISOCI,
1683         .descr = "Read from a disabled CCPI",
1684     },
1685     {
1686         .type  = ERR_UNCORRECTED,
1687         .mask  = L2C_CBC_INT_IOWRDISOCI,
1688         .descr = "Write to a disabled CCPI",
1689     },
1690     {0, 0, NULL},
1691 };
1692 
1693 #define L2C_CBC_INT_W1S     0x60008
1694 #define L2C_CBC_INT_ENA_W1C 0x60020
1695 
1696 #define L2C_CBC_INT_ENA_ALL  (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \
1697                   L2C_CBC_INT_IODISOCI)
1698 
1699 #define L2C_CBC_INT_ENA_W1S 0x60028
1700 
1701 #define L2C_CBC_IODISOCIERR 0x80008
1702 #define L2C_CBC_IOCERR      0x80010
1703 #define L2C_CBC_RSDERR      0x80018
1704 #define L2C_CBC_MIBERR      0x80020
1705 
1706 
1707 #define L2C_MCI_INT_W1C     0x0
1708 
1709 #define L2C_MCI_INT_VBFSBE   BIT(0)
1710 #define L2C_MCI_INT_VBFDBE   BIT(1)
1711 
1712 static const struct error_descr l2_mci_errors[] = {
1713     {
1714         .type  = ERR_CORRECTED,
1715         .mask  = L2C_MCI_INT_VBFSBE,
1716         .descr = "VBF single-bit error",
1717     },
1718     {
1719         .type  = ERR_UNCORRECTED,
1720         .mask  = L2C_MCI_INT_VBFDBE,
1721         .descr = "VBF double-bit error",
1722     },
1723     {0, 0, NULL},
1724 };
1725 
1726 #define L2C_MCI_INT_W1S     0x8
1727 #define L2C_MCI_INT_ENA_W1C 0x20
1728 
1729 #define L2C_MCI_INT_ENA_ALL  (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE)
1730 
1731 #define L2C_MCI_INT_ENA_W1S 0x28
1732 
1733 #define L2C_MCI_ERR     0x10000
1734 
1735 #define L2C_MESSAGE_SIZE    SZ_1K
1736 #define L2C_OTHER_SIZE      (50 * ARRAY_SIZE(l2_tad_errors))
1737 
1738 struct l2c_err_ctx {
1739     char *reg_ext_name;
1740     u64  reg_int;
1741     u64  reg_ext;
1742 };
1743 
1744 struct thunderx_l2c {
1745     void __iomem *regs;
1746     struct pci_dev *pdev;
1747     struct edac_device_ctl_info *edac_dev;
1748 
1749     struct dentry *debugfs;
1750 
1751     int index;
1752 
1753     struct msix_entry msix_ent;
1754 
1755     struct l2c_err_ctx err_ctx[RING_ENTRIES];
1756     unsigned long ring_head;
1757     unsigned long ring_tail;
1758 };
1759 
1760 static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id)
1761 {
1762     struct msix_entry *msix = irq_id;
1763     struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,
1764                         msix_ent);
1765 
1766     unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));
1767     struct l2c_err_ctx *ctx = &tad->err_ctx[head];
1768 
1769     ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);
1770 
1771     if (ctx->reg_int & L2C_TAD_INT_ECC) {
1772         ctx->reg_ext_name = "TQD_ERR";
1773         ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);
1774     } else if (ctx->reg_int & L2C_TAD_INT_TAG) {
1775         ctx->reg_ext_name = "TTG_ERR";
1776         ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);
1777     } else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {
1778         ctx->reg_ext_name = "TIMEOUT";
1779         ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);
1780     } else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {
1781         ctx->reg_ext_name = "ERR";
1782         ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
1783     }
1784 
1785     writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);
1786 
1787     tad->ring_head++;
1788 
1789     return IRQ_WAKE_THREAD;
1790 }
1791 
1792 static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id)
1793 {
1794     struct msix_entry *msix = irq_id;
1795     struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c,
1796                         msix_ent);
1797 
1798     unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));
1799     struct l2c_err_ctx *ctx = &cbc->err_ctx[head];
1800 
1801     ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);
1802 
1803     if (ctx->reg_int & L2C_CBC_INT_RSD) {
1804         ctx->reg_ext_name = "RSDERR";
1805         ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);
1806     } else if (ctx->reg_int & L2C_CBC_INT_MIB) {
1807         ctx->reg_ext_name = "MIBERR";
1808         ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);
1809     } else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {
1810         ctx->reg_ext_name = "IODISOCIERR";
1811         ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);
1812     }
1813 
1814     writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);
1815 
1816     cbc->ring_head++;
1817 
1818     return IRQ_WAKE_THREAD;
1819 }
1820 
1821 static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id)
1822 {
1823     struct msix_entry *msix = irq_id;
1824     struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c,
1825                         msix_ent);
1826 
1827     unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));
1828     struct l2c_err_ctx *ctx = &mci->err_ctx[head];
1829 
1830     ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);
1831     ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);
1832 
1833     writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);
1834 
1835     ctx->reg_ext_name = "ERR";
1836 
1837     mci->ring_head++;
1838 
1839     return IRQ_WAKE_THREAD;
1840 }
1841 
1842 static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id)
1843 {
1844     struct msix_entry *msix = irq_id;
1845     struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c,
1846                         msix_ent);
1847 
1848     unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));
1849     struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];
1850     irqreturn_t ret = IRQ_NONE;
1851 
1852     u64 mask_ue, mask_ce;
1853     const struct error_descr *l2_errors;
1854     char *reg_int_name;
1855 
1856     char *msg;
1857     char *other;
1858 
1859     msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1860     other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1861 
1862     if (!msg || !other)
1863         goto err_free;
1864 
1865     switch (l2c->pdev->device) {
1866     case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1867         reg_int_name = "L2C_TAD_INT";
1868         mask_ue = L2C_TAD_INT_UE;
1869         mask_ce = L2C_TAD_INT_CE;
1870         l2_errors = l2_tad_errors;
1871         break;
1872     case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1873         reg_int_name = "L2C_CBC_INT";
1874         mask_ue = L2C_CBC_INT_UE;
1875         mask_ce = L2C_CBC_INT_CE;
1876         l2_errors = l2_cbc_errors;
1877         break;
1878     case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1879         reg_int_name = "L2C_MCI_INT";
1880         mask_ue = L2C_MCI_INT_VBFDBE;
1881         mask_ce = L2C_MCI_INT_VBFSBE;
1882         l2_errors = l2_mci_errors;
1883         break;
1884     default:
1885         dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",
1886             l2c->pdev->device);
1887         goto err_free;
1888     }
1889 
1890     while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,
1891             ARRAY_SIZE(l2c->err_ctx))) {
1892         snprintf(msg, L2C_MESSAGE_SIZE,
1893              "%s: %s: %016llx, %s: %016llx",
1894              l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,
1895              ctx->reg_ext_name, ctx->reg_ext);
1896 
1897         decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);
1898 
1899         strncat(msg, other, L2C_MESSAGE_SIZE);
1900 
1901         if (ctx->reg_int & mask_ue)
1902             edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);
1903         else if (ctx->reg_int & mask_ce)
1904             edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);
1905 
1906         l2c->ring_tail++;
1907     }
1908 
1909     ret = IRQ_HANDLED;
1910 
1911 err_free:
1912     kfree(other);
1913     kfree(msg);
1914 
1915     return ret;
1916 }
1917 
1918 #define L2C_DEBUGFS_ATTR(_name, _reg)   DEBUGFS_REG_ATTR(l2c, _name, _reg)
1919 
1920 L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S);
1921 
1922 static struct debugfs_entry *l2c_tad_dfs_ents[] = {
1923     &debugfs_tad_int,
1924 };
1925 
1926 L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S);
1927 
1928 static struct debugfs_entry *l2c_cbc_dfs_ents[] = {
1929     &debugfs_cbc_int,
1930 };
1931 
1932 L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S);
1933 
1934 static struct debugfs_entry *l2c_mci_dfs_ents[] = {
1935     &debugfs_mci_int,
1936 };
1937 
1938 static const struct pci_device_id thunderx_l2c_pci_tbl[] = {
1939     { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), },
1940     { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), },
1941     { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), },
1942     { 0, },
1943 };
1944 
1945 static int thunderx_l2c_probe(struct pci_dev *pdev,
1946                   const struct pci_device_id *id)
1947 {
1948     struct thunderx_l2c *l2c;
1949     struct edac_device_ctl_info *edac_dev;
1950     struct debugfs_entry **l2c_devattr;
1951     size_t dfs_entries;
1952     irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL;
1953     char name[32];
1954     const char *fmt;
1955     u64 reg_en_offs, reg_en_mask;
1956     int idx;
1957     int ret;
1958 
1959     ret = pcim_enable_device(pdev);
1960     if (ret) {
1961         dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1962         return ret;
1963     }
1964 
1965     ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c");
1966     if (ret) {
1967         dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1968         return ret;
1969     }
1970 
1971     switch (pdev->device) {
1972     case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1973         thunderx_l2c_isr = thunderx_l2c_tad_isr;
1974         l2c_devattr = l2c_tad_dfs_ents;
1975         dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents);
1976         fmt = "L2C-TAD%d";
1977         reg_en_offs = L2C_TAD_INT_ENA_W1S;
1978         reg_en_mask = L2C_TAD_INT_ENA_ALL;
1979         break;
1980     case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1981         thunderx_l2c_isr = thunderx_l2c_cbc_isr;
1982         l2c_devattr = l2c_cbc_dfs_ents;
1983         dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents);
1984         fmt = "L2C-CBC%d";
1985         reg_en_offs = L2C_CBC_INT_ENA_W1S;
1986         reg_en_mask = L2C_CBC_INT_ENA_ALL;
1987         break;
1988     case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1989         thunderx_l2c_isr = thunderx_l2c_mci_isr;
1990         l2c_devattr = l2c_mci_dfs_ents;
1991         dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents);
1992         fmt = "L2C-MCI%d";
1993         reg_en_offs = L2C_MCI_INT_ENA_W1S;
1994         reg_en_mask = L2C_MCI_INT_ENA_ALL;
1995         break;
1996     default:
1997         //Should never ever get here
1998         dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",
1999             pdev->device);
2000         return -EINVAL;
2001     }
2002 
2003     idx = edac_device_alloc_index();
2004     snprintf(name, sizeof(name), fmt, idx);
2005 
2006     edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c),
2007                           name, 1, "L2C", 1, 0,
2008                           NULL, 0, idx);
2009     if (!edac_dev) {
2010         dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
2011         return -ENOMEM;
2012     }
2013 
2014     l2c = edac_dev->pvt_info;
2015     l2c->edac_dev = edac_dev;
2016 
2017     l2c->regs = pcim_iomap_table(pdev)[0];
2018     if (!l2c->regs) {
2019         dev_err(&pdev->dev, "Cannot map PCI resources\n");
2020         ret = -ENODEV;
2021         goto err_free;
2022     }
2023 
2024     l2c->pdev = pdev;
2025 
2026     l2c->ring_head = 0;
2027     l2c->ring_tail = 0;
2028 
2029     l2c->msix_ent.entry = 0;
2030     l2c->msix_ent.vector = 0;
2031 
2032     ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);
2033     if (ret) {
2034         dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
2035         goto err_free;
2036     }
2037 
2038     ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,
2039                     thunderx_l2c_isr,
2040                     thunderx_l2c_threaded_isr,
2041                     0, "[EDAC] ThunderX L2C",
2042                     &l2c->msix_ent);
2043     if (ret)
2044         goto err_free;
2045 
2046     edac_dev->dev = &pdev->dev;
2047     edac_dev->dev_name = dev_name(&pdev->dev);
2048     edac_dev->mod_name = "thunderx-l2c";
2049     edac_dev->ctl_name = "thunderx-l2c";
2050 
2051     ret = edac_device_add_device(edac_dev);
2052     if (ret) {
2053         dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
2054         goto err_free;
2055     }
2056 
2057     if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
2058         l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
2059 
2060         ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,
2061                           l2c, dfs_entries);
2062 
2063         if (ret != dfs_entries) {
2064             dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
2065                  ret, ret >= 0 ? " created" : "");
2066         }
2067     }
2068 
2069     pci_set_drvdata(pdev, edac_dev);
2070 
2071     writeq(reg_en_mask, l2c->regs + reg_en_offs);
2072 
2073     return 0;
2074 
2075 err_free:
2076     edac_device_free_ctl_info(edac_dev);
2077 
2078     return ret;
2079 }
2080 
2081 static void thunderx_l2c_remove(struct pci_dev *pdev)
2082 {
2083     struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
2084     struct thunderx_l2c *l2c = edac_dev->pvt_info;
2085 
2086     switch (pdev->device) {
2087     case PCI_DEVICE_ID_THUNDER_L2C_TAD:
2088         writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);
2089         break;
2090     case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2091         writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);
2092         break;
2093     case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2094         writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);
2095         break;
2096     }
2097 
2098     edac_debugfs_remove_recursive(l2c->debugfs);
2099 
2100     edac_device_del_device(&pdev->dev);
2101     edac_device_free_ctl_info(edac_dev);
2102 }
2103 
2104 MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl);
2105 
2106 static struct pci_driver thunderx_l2c_driver = {
2107     .name     = "thunderx_l2c_edac",
2108     .probe    = thunderx_l2c_probe,
2109     .remove   = thunderx_l2c_remove,
2110     .id_table = thunderx_l2c_pci_tbl,
2111 };
2112 
2113 static int __init thunderx_edac_init(void)
2114 {
2115     int rc = 0;
2116 
2117     rc = pci_register_driver(&thunderx_lmc_driver);
2118     if (rc)
2119         return rc;
2120 
2121     rc = pci_register_driver(&thunderx_ocx_driver);
2122     if (rc)
2123         goto err_lmc;
2124 
2125     rc = pci_register_driver(&thunderx_l2c_driver);
2126     if (rc)
2127         goto err_ocx;
2128 
2129     return rc;
2130 err_ocx:
2131     pci_unregister_driver(&thunderx_ocx_driver);
2132 err_lmc:
2133     pci_unregister_driver(&thunderx_lmc_driver);
2134 
2135     return rc;
2136 }
2137 
2138 static void __exit thunderx_edac_exit(void)
2139 {
2140     pci_unregister_driver(&thunderx_l2c_driver);
2141     pci_unregister_driver(&thunderx_ocx_driver);
2142     pci_unregister_driver(&thunderx_lmc_driver);
2143 
2144 }
2145 
2146 module_init(thunderx_edac_init);
2147 module_exit(thunderx_edac_exit);
2148 
2149 MODULE_LICENSE("GPL v2");
2150 MODULE_AUTHOR("Cavium, Inc.");
2151 MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!