mips/kernel/mips-cm.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Copyright (C) 2013 Imagination Technologies
0004  * Author: Paul Burton <paul.burton@mips.com>
0005  */
0006
0007 #include <linux/errno.h>
0008 #include <linux/percpu.h>
0009 #include <linux/spinlock.h>
0010
0011 #include <asm/mips-cps.h>
0012 #include <asm/mipsregs.h>
0013
0014 void __iomem *mips_gcr_base;
0015 void __iomem *mips_cm_l2sync_base;
0016 int mips_cm_is64;
0017
0018 static char *cm2_tr[8] = {
0019     "mem",  "gcr",  "gic",  "mmio",
0020     "0x04", "cpc", "0x06", "0x07"
0021 };
0022
0023 /* CM3 Tag ECC transaction type */
0024 static char *cm3_tr[16] = {
0025     [0x0] = "ReqNoData",
0026     [0x1] = "0x1",
0027     [0x2] = "ReqWData",
0028     [0x3] = "0x3",
0029     [0x4] = "IReqNoResp",
0030     [0x5] = "IReqWResp",
0031     [0x6] = "IReqNoRespDat",
0032     [0x7] = "IReqWRespDat",
0033     [0x8] = "RespNoData",
0034     [0x9] = "RespDataFol",
0035     [0xa] = "RespWData",
0036     [0xb] = "RespDataOnly",
0037     [0xc] = "IRespNoData",
0038     [0xd] = "IRespDataFol",
0039     [0xe] = "IRespWData",
0040     [0xf] = "IRespDataOnly"
0041 };
0042
0043 static char *cm2_cmd[32] = {
0044     [0x00] = "0x00",
0045     [0x01] = "Legacy Write",
0046     [0x02] = "Legacy Read",
0047     [0x03] = "0x03",
0048     [0x04] = "0x04",
0049     [0x05] = "0x05",
0050     [0x06] = "0x06",
0051     [0x07] = "0x07",
0052     [0x08] = "Coherent Read Own",
0053     [0x09] = "Coherent Read Share",
0054     [0x0a] = "Coherent Read Discard",
0055     [0x0b] = "Coherent Ready Share Always",
0056     [0x0c] = "Coherent Upgrade",
0057     [0x0d] = "Coherent Writeback",
0058     [0x0e] = "0x0e",
0059     [0x0f] = "0x0f",
0060     [0x10] = "Coherent Copyback",
0061     [0x11] = "Coherent Copyback Invalidate",
0062     [0x12] = "Coherent Invalidate",
0063     [0x13] = "Coherent Write Invalidate",
0064     [0x14] = "Coherent Completion Sync",
0065     [0x15] = "0x15",
0066     [0x16] = "0x16",
0067     [0x17] = "0x17",
0068     [0x18] = "0x18",
0069     [0x19] = "0x19",
0070     [0x1a] = "0x1a",
0071     [0x1b] = "0x1b",
0072     [0x1c] = "0x1c",
0073     [0x1d] = "0x1d",
0074     [0x1e] = "0x1e",
0075     [0x1f] = "0x1f"
0076 };
0077
0078 /* CM3 Tag ECC command type */
0079 static char *cm3_cmd[16] = {
0080     [0x0] = "Legacy Read",
0081     [0x1] = "Legacy Write",
0082     [0x2] = "Coherent Read Own",
0083     [0x3] = "Coherent Read Share",
0084     [0x4] = "Coherent Read Discard",
0085     [0x5] = "Coherent Evicted",
0086     [0x6] = "Coherent Upgrade",
0087     [0x7] = "Coherent Upgrade for Store Conditional",
0088     [0x8] = "Coherent Writeback",
0089     [0x9] = "Coherent Write Invalidate",
0090     [0xa] = "0xa",
0091     [0xb] = "0xb",
0092     [0xc] = "0xc",
0093     [0xd] = "0xd",
0094     [0xe] = "0xe",
0095     [0xf] = "0xf"
0096 };
0097
0098 /* CM3 Tag ECC command group */
0099 static char *cm3_cmd_group[8] = {
0100     [0x0] = "Normal",
0101     [0x1] = "Registers",
0102     [0x2] = "TLB",
0103     [0x3] = "0x3",
0104     [0x4] = "L1I",
0105     [0x5] = "L1D",
0106     [0x6] = "L3",
0107     [0x7] = "L2"
0108 };
0109
0110 static char *cm2_core[8] = {
0111     "Invalid/OK",   "Invalid/Data",
0112     "Shared/OK",    "Shared/Data",
0113     "Modified/OK",  "Modified/Data",
0114     "Exclusive/OK", "Exclusive/Data"
0115 };
0116
0117 static char *cm2_l2_type[4] = {
0118     [0x0] = "None",
0119     [0x1] = "Tag RAM single/double ECC error",
0120     [0x2] = "Data RAM single/double ECC error",
0121     [0x3] = "WS RAM uncorrectable dirty parity"
0122 };
0123
0124 static char *cm2_l2_instr[32] = {
0125     [0x00] = "L2_NOP",
0126     [0x01] = "L2_ERR_CORR",
0127     [0x02] = "L2_TAG_INV",
0128     [0x03] = "L2_WS_CLEAN",
0129     [0x04] = "L2_RD_MDYFY_WR",
0130     [0x05] = "L2_WS_MRU",
0131     [0x06] = "L2_EVICT_LN2",
0132     [0x07] = "0x07",
0133     [0x08] = "L2_EVICT",
0134     [0x09] = "L2_REFL",
0135     [0x0a] = "L2_RD",
0136     [0x0b] = "L2_WR",
0137     [0x0c] = "L2_EVICT_MRU",
0138     [0x0d] = "L2_SYNC",
0139     [0x0e] = "L2_REFL_ERR",
0140     [0x0f] = "0x0f",
0141     [0x10] = "L2_INDX_WB_INV",
0142     [0x11] = "L2_INDX_LD_TAG",
0143     [0x12] = "L2_INDX_ST_TAG",
0144     [0x13] = "L2_INDX_ST_DATA",
0145     [0x14] = "L2_INDX_ST_ECC",
0146     [0x15] = "0x15",
0147     [0x16] = "0x16",
0148     [0x17] = "0x17",
0149     [0x18] = "L2_FTCH_AND_LCK",
0150     [0x19] = "L2_HIT_INV",
0151     [0x1a] = "L2_HIT_WB_INV",
0152     [0x1b] = "L2_HIT_WB",
0153     [0x1c] = "0x1c",
0154     [0x1d] = "0x1d",
0155     [0x1e] = "0x1e",
0156     [0x1f] = "0x1f"
0157 };
0158
0159 static char *cm2_causes[32] = {
0160     "None", "GC_WR_ERR", "GC_RD_ERR", "COH_WR_ERR",
0161     "COH_RD_ERR", "MMIO_WR_ERR", "MMIO_RD_ERR", "0x07",
0162     "0x08", "0x09", "0x0a", "0x0b",
0163     "0x0c", "0x0d", "0x0e", "0x0f",
0164     "0x10", "INTVN_WR_ERR", "INTVN_RD_ERR", "0x13",
0165     "0x14", "0x15", "0x16", "0x17",
0166     "L2_RD_UNCORR", "L2_WR_UNCORR", "L2_CORR", "0x1b",
0167     "0x1c", "0x1d", "0x1e", "0x1f"
0168 };
0169
0170 static char *cm3_causes[32] = {
0171     "0x0", "MP_CORRECTABLE_ECC_ERR", "MP_REQUEST_DECODE_ERR",
0172     "MP_UNCORRECTABLE_ECC_ERR", "MP_PARITY_ERR", "MP_COHERENCE_ERR",
0173     "CMBIU_REQUEST_DECODE_ERR", "CMBIU_PARITY_ERR", "CMBIU_AXI_RESP_ERR",
0174     "0x9", "RBI_BUS_ERR", "0xb", "0xc", "0xd", "0xe", "0xf", "0x10",
0175     "0x11", "0x12", "0x13", "0x14", "0x15", "0x16", "0x17", "0x18",
0176     "0x19", "0x1a", "0x1b", "0x1c", "0x1d", "0x1e", "0x1f"
0177 };
0178
0179 static DEFINE_PER_CPU_ALIGNED(spinlock_t, cm_core_lock);
0180 static DEFINE_PER_CPU_ALIGNED(unsigned long, cm_core_lock_flags);
0181
0182 phys_addr_t __mips_cm_phys_base(void)
0183 {
0184     u32 config3 = read_c0_config3();
0185     unsigned long cmgcr;
0186
0187     /* Check the CMGCRBase register is implemented */
0188     if (!(config3 & MIPS_CONF3_CMGCR))
0189         return 0;
0190
0191     /* Read the address from CMGCRBase */
0192     cmgcr = read_c0_cmgcrbase();
0193     return (cmgcr & MIPS_CMGCRF_BASE) << (36 - 32);
0194 }
0195
0196 phys_addr_t mips_cm_phys_base(void)
0197     __attribute__((weak, alias("__mips_cm_phys_base")));
0198
0199 phys_addr_t __mips_cm_l2sync_phys_base(void)
0200 {
0201     u32 base_reg;
0202
0203     /*
0204      * If the L2-only sync region is already enabled then leave it at it's
0205      * current location.
0206      */
0207     base_reg = read_gcr_l2_only_sync_base();
0208     if (base_reg & CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN)
0209         return base_reg & CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE;
0210
0211     /* Default to following the CM */
0212     return mips_cm_phys_base() + MIPS_CM_GCR_SIZE;
0213 }
0214
0215 phys_addr_t mips_cm_l2sync_phys_base(void)
0216     __attribute__((weak, alias("__mips_cm_l2sync_phys_base")));
0217
0218 static void mips_cm_probe_l2sync(void)
0219 {
0220     unsigned major_rev;
0221     phys_addr_t addr;
0222
0223     /* L2-only sync was introduced with CM major revision 6 */
0224     major_rev = FIELD_GET(CM_GCR_REV_MAJOR, read_gcr_rev());
0225     if (major_rev < 6)
0226         return;
0227
0228     /* Find a location for the L2 sync region */
0229     addr = mips_cm_l2sync_phys_base();
0230     BUG_ON((addr & CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE) != addr);
0231     if (!addr)
0232         return;
0233
0234     /* Set the region base address & enable it */
0235     write_gcr_l2_only_sync_base(addr | CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN);
0236
0237     /* Map the region */
0238     mips_cm_l2sync_base = ioremap(addr, MIPS_CM_L2SYNC_SIZE);
0239 }
0240
0241 int mips_cm_probe(void)
0242 {
0243     phys_addr_t addr;
0244     u32 base_reg;
0245     unsigned cpu;
0246
0247     /*
0248      * No need to probe again if we have already been
0249      * here before.
0250      */
0251     if (mips_gcr_base)
0252         return 0;
0253
0254     addr = mips_cm_phys_base();
0255     BUG_ON((addr & CM_GCR_BASE_GCRBASE) != addr);
0256     if (!addr)
0257         return -ENODEV;
0258
0259     mips_gcr_base = ioremap(addr, MIPS_CM_GCR_SIZE);
0260     if (!mips_gcr_base)
0261         return -ENXIO;
0262
0263     /* sanity check that we're looking at a CM */
0264     base_reg = read_gcr_base();
0265     if ((base_reg & CM_GCR_BASE_GCRBASE) != addr) {
0266         pr_err("GCRs appear to have been moved (expected them at 0x%08lx)!\n",
0267                (unsigned long)addr);
0268         iounmap(mips_gcr_base);
0269         mips_gcr_base = NULL;
0270         return -ENODEV;
0271     }
0272
0273     /* set default target to memory */
0274     change_gcr_base(CM_GCR_BASE_CMDEFTGT, CM_GCR_BASE_CMDEFTGT_MEM);
0275
0276     /* disable CM regions */
0277     write_gcr_reg0_base(CM_GCR_REGn_BASE_BASEADDR);
0278     write_gcr_reg0_mask(CM_GCR_REGn_MASK_ADDRMASK);
0279     write_gcr_reg1_base(CM_GCR_REGn_BASE_BASEADDR);
0280     write_gcr_reg1_mask(CM_GCR_REGn_MASK_ADDRMASK);
0281     write_gcr_reg2_base(CM_GCR_REGn_BASE_BASEADDR);
0282     write_gcr_reg2_mask(CM_GCR_REGn_MASK_ADDRMASK);
0283     write_gcr_reg3_base(CM_GCR_REGn_BASE_BASEADDR);
0284     write_gcr_reg3_mask(CM_GCR_REGn_MASK_ADDRMASK);
0285
0286     /* probe for an L2-only sync region */
0287     mips_cm_probe_l2sync();
0288
0289     /* determine register width for this CM */
0290     mips_cm_is64 = IS_ENABLED(CONFIG_64BIT) && (mips_cm_revision() >= CM_REV_CM3);
0291
0292     for_each_possible_cpu(cpu)
0293         spin_lock_init(&per_cpu(cm_core_lock, cpu));
0294
0295     return 0;
0296 }
0297
0298 void mips_cm_lock_other(unsigned int cluster, unsigned int core,
0299             unsigned int vp, unsigned int block)
0300 {
0301     unsigned int curr_core, cm_rev;
0302     u32 val;
0303
0304     cm_rev = mips_cm_revision();
0305     preempt_disable();
0306
0307     if (cm_rev >= CM_REV_CM3) {
0308         val = FIELD_PREP(CM3_GCR_Cx_OTHER_CORE, core) |
0309               FIELD_PREP(CM3_GCR_Cx_OTHER_VP, vp);
0310
0311         if (cm_rev >= CM_REV_CM3_5) {
0312             val |= CM_GCR_Cx_OTHER_CLUSTER_EN;
0313             val |= FIELD_PREP(CM_GCR_Cx_OTHER_CLUSTER, cluster);
0314             val |= FIELD_PREP(CM_GCR_Cx_OTHER_BLOCK, block);
0315         } else {
0316             WARN_ON(cluster != 0);
0317             WARN_ON(block != CM_GCR_Cx_OTHER_BLOCK_LOCAL);
0318         }
0319
0320         /*
0321          * We need to disable interrupts in SMP systems in order to
0322          * ensure that we don't interrupt the caller with code which
0323          * may modify the redirect register. We do so here in a
0324          * slightly obscure way by using a spin lock, since this has
0325          * the neat property of also catching any nested uses of
0326          * mips_cm_lock_other() leading to a deadlock or a nice warning
0327          * with lockdep enabled.
0328          */
0329         spin_lock_irqsave(this_cpu_ptr(&cm_core_lock),
0330                   *this_cpu_ptr(&cm_core_lock_flags));
0331     } else {
0332         WARN_ON(cluster != 0);
0333         WARN_ON(block != CM_GCR_Cx_OTHER_BLOCK_LOCAL);
0334
0335         /*
0336          * We only have a GCR_CL_OTHER per core in systems with
0337          * CM 2.5 & older, so have to ensure other VP(E)s don't
0338          * race with us.
0339          */
0340         curr_core = cpu_core(&current_cpu_data);
0341         spin_lock_irqsave(&per_cpu(cm_core_lock, curr_core),
0342                   per_cpu(cm_core_lock_flags, curr_core));
0343
0344         val = FIELD_PREP(CM_GCR_Cx_OTHER_CORENUM, core);
0345     }
0346
0347     write_gcr_cl_other(val);
0348
0349     /*
0350      * Ensure the core-other region reflects the appropriate core &
0351      * VP before any accesses to it occur.
0352      */
0353     mb();
0354 }
0355
0356 void mips_cm_unlock_other(void)
0357 {
0358     unsigned int curr_core;
0359
0360     if (mips_cm_revision() < CM_REV_CM3) {
0361         curr_core = cpu_core(&current_cpu_data);
0362         spin_unlock_irqrestore(&per_cpu(cm_core_lock, curr_core),
0363                        per_cpu(cm_core_lock_flags, curr_core));
0364     } else {
0365         spin_unlock_irqrestore(this_cpu_ptr(&cm_core_lock),
0366                        *this_cpu_ptr(&cm_core_lock_flags));
0367     }
0368
0369     preempt_enable();
0370 }
0371
0372 void mips_cm_error_report(void)
0373 {
0374     u64 cm_error, cm_addr, cm_other;
0375     unsigned long revision;
0376     int ocause, cause;
0377     char buf[256];
0378
0379     if (!mips_cm_present())
0380         return;
0381
0382     revision = mips_cm_revision();
0383     cm_error = read_gcr_error_cause();
0384     cm_addr = read_gcr_error_addr();
0385     cm_other = read_gcr_error_mult();
0386
0387     if (revision < CM_REV_CM3) { /* CM2 */
0388         cause = FIELD_GET(CM_GCR_ERROR_CAUSE_ERRTYPE, cm_error);
0389         ocause = FIELD_GET(CM_GCR_ERROR_MULT_ERR2ND, cm_other);
0390
0391         if (!cause)
0392             return;
0393
0394         if (cause < 16) {
0395             unsigned long cca_bits = (cm_error >> 15) & 7;
0396             unsigned long tr_bits = (cm_error >> 12) & 7;
0397             unsigned long cmd_bits = (cm_error >> 7) & 0x1f;
0398             unsigned long stag_bits = (cm_error >> 3) & 15;
0399             unsigned long sport_bits = (cm_error >> 0) & 7;
0400
0401             snprintf(buf, sizeof(buf),
0402                  "CCA=%lu TR=%s MCmd=%s STag=%lu "
0403                  "SPort=%lu\n", cca_bits, cm2_tr[tr_bits],
0404                  cm2_cmd[cmd_bits], stag_bits, sport_bits);
0405         } else if (cause < 24) {
0406             /* glob state & sresp together */
0407             unsigned long c3_bits = (cm_error >> 18) & 7;
0408             unsigned long c2_bits = (cm_error >> 15) & 7;
0409             unsigned long c1_bits = (cm_error >> 12) & 7;
0410             unsigned long c0_bits = (cm_error >> 9) & 7;
0411             unsigned long sc_bit = (cm_error >> 8) & 1;
0412             unsigned long cmd_bits = (cm_error >> 3) & 0x1f;
0413             unsigned long sport_bits = (cm_error >> 0) & 7;
0414
0415             snprintf(buf, sizeof(buf),
0416                  "C3=%s C2=%s C1=%s C0=%s SC=%s "
0417                  "MCmd=%s SPort=%lu\n",
0418                  cm2_core[c3_bits], cm2_core[c2_bits],
0419                  cm2_core[c1_bits], cm2_core[c0_bits],
0420                  sc_bit ? "True" : "False",
0421                  cm2_cmd[cmd_bits], sport_bits);
0422         } else {
0423             unsigned long muc_bit = (cm_error >> 23) & 1;
0424             unsigned long ins_bits = (cm_error >> 18) & 0x1f;
0425             unsigned long arr_bits = (cm_error >> 16) & 3;
0426             unsigned long dw_bits = (cm_error >> 12) & 15;
0427             unsigned long way_bits = (cm_error >> 9) & 7;
0428             unsigned long mway_bit = (cm_error >> 8) & 1;
0429             unsigned long syn_bits = (cm_error >> 0) & 0xFF;
0430
0431             snprintf(buf, sizeof(buf),
0432                  "Type=%s%s Instr=%s DW=%lu Way=%lu "
0433                  "MWay=%s Syndrome=0x%02lx",
0434                  muc_bit ? "Multi-UC " : "",
0435                  cm2_l2_type[arr_bits],
0436                  cm2_l2_instr[ins_bits], dw_bits, way_bits,
0437                  mway_bit ? "True" : "False", syn_bits);
0438         }
0439         pr_err("CM_ERROR=%08llx %s <%s>\n", cm_error,
0440                cm2_causes[cause], buf);
0441         pr_err("CM_ADDR =%08llx\n", cm_addr);
0442         pr_err("CM_OTHER=%08llx %s\n", cm_other, cm2_causes[ocause]);
0443     } else { /* CM3 */
0444         ulong core_id_bits, vp_id_bits, cmd_bits, cmd_group_bits;
0445         ulong cm3_cca_bits, mcp_bits, cm3_tr_bits, sched_bit;
0446
0447         cause = FIELD_GET(CM3_GCR_ERROR_CAUSE_ERRTYPE, cm_error);
0448         ocause = FIELD_GET(CM_GCR_ERROR_MULT_ERR2ND, cm_other);
0449
0450         if (!cause)
0451             return;
0452
0453         /* Used by cause == {1,2,3} */
0454         core_id_bits = (cm_error >> 22) & 0xf;
0455         vp_id_bits = (cm_error >> 18) & 0xf;
0456         cmd_bits = (cm_error >> 14) & 0xf;
0457         cmd_group_bits = (cm_error >> 11) & 0xf;
0458         cm3_cca_bits = (cm_error >> 8) & 7;
0459         mcp_bits = (cm_error >> 5) & 0xf;
0460         cm3_tr_bits = (cm_error >> 1) & 0xf;
0461         sched_bit = cm_error & 0x1;
0462
0463         if (cause == 1 || cause == 3) { /* Tag ECC */
0464             unsigned long tag_ecc = (cm_error >> 57) & 0x1;
0465             unsigned long tag_way_bits = (cm_error >> 29) & 0xffff;
0466             unsigned long dword_bits = (cm_error >> 49) & 0xff;
0467             unsigned long data_way_bits = (cm_error >> 45) & 0xf;
0468             unsigned long data_sets_bits = (cm_error >> 29) & 0xfff;
0469             unsigned long bank_bit = (cm_error >> 28) & 0x1;
0470             snprintf(buf, sizeof(buf),
0471                  "%s ECC Error: Way=%lu (DWORD=%lu, Sets=%lu)"
0472                  "Bank=%lu CoreID=%lu VPID=%lu Command=%s"
0473                  "Command Group=%s CCA=%lu MCP=%d"
0474                  "Transaction type=%s Scheduler=%lu\n",
0475                  tag_ecc ? "TAG" : "DATA",
0476                  tag_ecc ? (unsigned long)ffs(tag_way_bits) - 1 :
0477                  data_way_bits, bank_bit, dword_bits,
0478                  data_sets_bits,
0479                  core_id_bits, vp_id_bits,
0480                  cm3_cmd[cmd_bits],
0481                  cm3_cmd_group[cmd_group_bits],
0482                  cm3_cca_bits, 1 << mcp_bits,
0483                  cm3_tr[cm3_tr_bits], sched_bit);
0484         } else if (cause == 2) {
0485             unsigned long data_error_type = (cm_error >> 41) & 0xfff;
0486             unsigned long data_decode_cmd = (cm_error >> 37) & 0xf;
0487             unsigned long data_decode_group = (cm_error >> 34) & 0x7;
0488             unsigned long data_decode_destination_id = (cm_error >> 28) & 0x3f;
0489
0490             snprintf(buf, sizeof(buf),
0491                  "Decode Request Error: Type=%lu, Command=%lu"
0492                  "Command Group=%lu Destination ID=%lu"
0493                  "CoreID=%lu VPID=%lu Command=%s"
0494                  "Command Group=%s CCA=%lu MCP=%d"
0495                  "Transaction type=%s Scheduler=%lu\n",
0496                  data_error_type, data_decode_cmd,
0497                  data_decode_group, data_decode_destination_id,
0498                  core_id_bits, vp_id_bits,
0499                  cm3_cmd[cmd_bits],
0500                  cm3_cmd_group[cmd_group_bits],
0501                  cm3_cca_bits, 1 << mcp_bits,
0502                  cm3_tr[cm3_tr_bits], sched_bit);
0503         } else {
0504             buf[0] = 0;
0505         }
0506
0507         pr_err("CM_ERROR=%llx %s <%s>\n", cm_error,
0508                cm3_causes[cause], buf);
0509         pr_err("CM_ADDR =%llx\n", cm_addr);
0510         pr_err("CM_OTHER=%llx %s\n", cm_other, cm3_causes[ocause]);
0511     }
0512
0513     /* reprime cause register */
0514     write_gcr_error_cause(cm_error);
0515 }