powerpc/perf/power9-pmu.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  * Performance counter support for POWER9 processors.
0004  *
0005  * Copyright 2009 Paul Mackerras, IBM Corporation.
0006  * Copyright 2013 Michael Ellerman, IBM Corporation.
0007  * Copyright 2016 Madhavan Srinivasan, IBM Corporation.
0008  */
0009
0010 #define pr_fmt(fmt) "power9-pmu: " fmt
0011
0012 #include "isa207-common.h"
0013
0014 /*
0015  * Raw event encoding for Power9:
0016  *
0017  *        60        56        52        48        44        40        36        32
0018  * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
0019  *   | | [ ]                       [ ] [      thresh_cmp     ]   [  thresh_ctl   ]
0020  *   | |  |                         |                                     |
0021  *   | |  *- IFM (Linux)            |                  thresh start/stop -*
0022  *   | *- BHRB (Linux)              *sm
0023  *   *- EBB (Linux)
0024  *
0025  *        28        24        20        16        12         8         4         0
0026  * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - |
0027  *   [   ] [  sample ]   [cache]   [ pmc ]   [unit ]   []    m   [    pmcxsel    ]
0028  *     |        |           |                          |     |
0029  *     |        |           |                          |     *- mark
0030  *     |        |           *- L1/L2/L3 cache_sel      |
0031  *     |        |                                      |
0032  *     |        *- sampling mode for marked events     *- combine
0033  *     |
0034  *     *- thresh_sel
0035  *
0036  * Below uses IBM bit numbering.
0037  *
0038  * MMCR1[x:y] = unit    (PMCxUNIT)
0039  * MMCR1[24]   = pmc1combine[0]
0040  * MMCR1[25]   = pmc1combine[1]
0041  * MMCR1[26]   = pmc2combine[0]
0042  * MMCR1[27]   = pmc2combine[1]
0043  * MMCR1[28]   = pmc3combine[0]
0044  * MMCR1[29]   = pmc3combine[1]
0045  * MMCR1[30]   = pmc4combine[0]
0046  * MMCR1[31]   = pmc4combine[1]
0047  *
0048  * if pmc == 3 and unit == 0 and pmcxsel[0:6] == 0b0101011
0049  *  MMCR1[20:27] = thresh_ctl
0050  * else if pmc == 4 and unit == 0xf and pmcxsel[0:6] == 0b0101001
0051  *  MMCR1[20:27] = thresh_ctl
0052  * else
0053  *  MMCRA[48:55] = thresh_ctl   (THRESH START/END)
0054  *
0055  * if thresh_sel:
0056  *  MMCRA[45:47] = thresh_sel
0057  *
0058  * if thresh_cmp:
0059  *  MMCRA[9:11] = thresh_cmp[0:2]
0060  *  MMCRA[12:18] = thresh_cmp[3:9]
0061  *
0062  * MMCR1[16] = cache_sel[2]
0063  * MMCR1[17] = cache_sel[3]
0064  *
0065  * if mark:
0066  *  MMCRA[63]    = 1        (SAMPLE_ENABLE)
0067  *  MMCRA[57:59] = sample[0:2]  (RAND_SAMP_ELIG)
0068  * MMCRA[61:62] = sample[3:4]  (RAND_SAMP_MODE)
0069  *
0070  * if EBB and BHRB:
0071  *  MMCRA[32:33] = IFM
0072  *
0073  * MMCRA[SDAR_MODE]  = sm
0074  */
0075
0076 /*
0077  * Some power9 event codes.
0078  */
0079 #define EVENT(_name, _code) _name = _code,
0080
0081 enum {
0082 #include "power9-events-list.h"
0083 };
0084
0085 #undef EVENT
0086
0087 /* MMCRA IFM bits - POWER9 */
0088 #define POWER9_MMCRA_IFM1       0x0000000040000000UL
0089 #define POWER9_MMCRA_IFM2       0x0000000080000000UL
0090 #define POWER9_MMCRA_IFM3       0x00000000C0000000UL
0091 #define POWER9_MMCRA_BHRB_MASK      0x00000000C0000000UL
0092
0093 extern u64 PERF_REG_EXTENDED_MASK;
0094
0095 /* Nasty Power9 specific hack */
0096 #define PVR_POWER9_CUMULUS      0x00002000
0097
0098 /* PowerISA v2.07 format attribute structure*/
0099 extern const struct attribute_group isa207_pmu_format_group;
0100
0101 static int p9_dd21_bl_ev[] = {
0102     PM_MRK_ST_DONE_L2,
0103     PM_RADIX_PWC_L1_HIT,
0104     PM_FLOP_CMPL,
0105     PM_MRK_NTF_FIN,
0106     PM_RADIX_PWC_L2_HIT,
0107     PM_IFETCH_THROTTLE,
0108     PM_MRK_L2_TM_ST_ABORT_SISTER,
0109     PM_RADIX_PWC_L3_HIT,
0110     PM_RUN_CYC_SMT2_MODE,
0111     PM_TM_TX_PASS_RUN_INST,
0112     PM_DISP_HELD_SYNC_HOLD,
0113 };
0114
0115 static int p9_dd22_bl_ev[] = {
0116     PM_DTLB_MISS_16G,
0117     PM_DERAT_MISS_2M,
0118     PM_DTLB_MISS_2M,
0119     PM_MRK_DTLB_MISS_1G,
0120     PM_DTLB_MISS_4K,
0121     PM_DERAT_MISS_1G,
0122     PM_MRK_DERAT_MISS_2M,
0123     PM_MRK_DTLB_MISS_4K,
0124     PM_MRK_DTLB_MISS_16G,
0125     PM_DTLB_MISS_64K,
0126     PM_MRK_DERAT_MISS_1G,
0127     PM_MRK_DTLB_MISS_64K,
0128     PM_DISP_HELD_SYNC_HOLD,
0129     PM_DTLB_MISS_16M,
0130     PM_DTLB_MISS_1G,
0131     PM_MRK_DTLB_MISS_16M,
0132 };
0133
0134 /* Table of alternatives, sorted by column 0 */
0135 static const unsigned int power9_event_alternatives[][MAX_ALT] = {
0136     { PM_BR_2PATH,          PM_BR_2PATH_ALT },
0137     { PM_INST_DISP,         PM_INST_DISP_ALT },
0138     { PM_RUN_CYC_ALT,               PM_RUN_CYC },
0139     { PM_LD_MISS_L1,                PM_LD_MISS_L1_ALT },
0140     { PM_RUN_INST_CMPL_ALT,         PM_RUN_INST_CMPL },
0141 };
0142
0143 static int power9_get_alternatives(u64 event, unsigned int flags, u64 alt[])
0144 {
0145     int num_alt = 0;
0146
0147     num_alt = isa207_get_alternatives(event, alt,
0148                       ARRAY_SIZE(power9_event_alternatives), flags,
0149                       power9_event_alternatives);
0150
0151     return num_alt;
0152 }
0153
0154 static int power9_check_attr_config(struct perf_event *ev)
0155 {
0156     u64 val;
0157     u64 event = ev->attr.config;
0158
0159     val = (event >> EVENT_SAMPLE_SHIFT) & EVENT_SAMPLE_MASK;
0160     if (val == 0xC || isa3XX_check_attr_config(ev))
0161         return -EINVAL;
0162
0163     return 0;
0164 }
0165
0166 GENERIC_EVENT_ATTR(cpu-cycles,          PM_CYC);
0167 GENERIC_EVENT_ATTR(stalled-cycles-frontend, PM_ICT_NOSLOT_CYC);
0168 GENERIC_EVENT_ATTR(stalled-cycles-backend,  PM_CMPLU_STALL);
0169 GENERIC_EVENT_ATTR(instructions,        PM_INST_CMPL);
0170 GENERIC_EVENT_ATTR(branch-instructions,     PM_BR_CMPL);
0171 GENERIC_EVENT_ATTR(branch-misses,       PM_BR_MPRED_CMPL);
0172 GENERIC_EVENT_ATTR(cache-references,        PM_LD_REF_L1);
0173 GENERIC_EVENT_ATTR(cache-misses,        PM_LD_MISS_L1_FIN);
0174 GENERIC_EVENT_ATTR(mem-loads,           MEM_LOADS);
0175 GENERIC_EVENT_ATTR(mem-stores,          MEM_STORES);
0176
0177 CACHE_EVENT_ATTR(L1-dcache-load-misses,     PM_LD_MISS_L1_FIN);
0178 CACHE_EVENT_ATTR(L1-dcache-loads,       PM_LD_REF_L1);
0179 CACHE_EVENT_ATTR(L1-dcache-prefetches,      PM_L1_PREF);
0180 CACHE_EVENT_ATTR(L1-dcache-store-misses,    PM_ST_MISS_L1);
0181 CACHE_EVENT_ATTR(L1-icache-load-misses,     PM_L1_ICACHE_MISS);
0182 CACHE_EVENT_ATTR(L1-icache-loads,       PM_INST_FROM_L1);
0183 CACHE_EVENT_ATTR(L1-icache-prefetches,      PM_IC_PREF_WRITE);
0184 CACHE_EVENT_ATTR(LLC-load-misses,       PM_DATA_FROM_L3MISS);
0185 CACHE_EVENT_ATTR(LLC-loads,         PM_DATA_FROM_L3);
0186 CACHE_EVENT_ATTR(LLC-prefetches,        PM_L3_PREF_ALL);
0187 CACHE_EVENT_ATTR(branch-load-misses,        PM_BR_MPRED_CMPL);
0188 CACHE_EVENT_ATTR(branch-loads,          PM_BR_CMPL);
0189 CACHE_EVENT_ATTR(dTLB-load-misses,      PM_DTLB_MISS);
0190 CACHE_EVENT_ATTR(iTLB-load-misses,      PM_ITLB_MISS);
0191
0192 static struct attribute *power9_events_attr[] = {
0193     GENERIC_EVENT_PTR(PM_CYC),
0194     GENERIC_EVENT_PTR(PM_ICT_NOSLOT_CYC),
0195     GENERIC_EVENT_PTR(PM_CMPLU_STALL),
0196     GENERIC_EVENT_PTR(PM_INST_CMPL),
0197     GENERIC_EVENT_PTR(PM_BR_CMPL),
0198     GENERIC_EVENT_PTR(PM_BR_MPRED_CMPL),
0199     GENERIC_EVENT_PTR(PM_LD_REF_L1),
0200     GENERIC_EVENT_PTR(PM_LD_MISS_L1_FIN),
0201     GENERIC_EVENT_PTR(MEM_LOADS),
0202     GENERIC_EVENT_PTR(MEM_STORES),
0203     CACHE_EVENT_PTR(PM_LD_MISS_L1_FIN),
0204     CACHE_EVENT_PTR(PM_LD_REF_L1),
0205     CACHE_EVENT_PTR(PM_L1_PREF),
0206     CACHE_EVENT_PTR(PM_ST_MISS_L1),
0207     CACHE_EVENT_PTR(PM_L1_ICACHE_MISS),
0208     CACHE_EVENT_PTR(PM_INST_FROM_L1),
0209     CACHE_EVENT_PTR(PM_IC_PREF_WRITE),
0210     CACHE_EVENT_PTR(PM_DATA_FROM_L3MISS),
0211     CACHE_EVENT_PTR(PM_DATA_FROM_L3),
0212     CACHE_EVENT_PTR(PM_L3_PREF_ALL),
0213     CACHE_EVENT_PTR(PM_BR_MPRED_CMPL),
0214     CACHE_EVENT_PTR(PM_BR_CMPL),
0215     CACHE_EVENT_PTR(PM_DTLB_MISS),
0216     CACHE_EVENT_PTR(PM_ITLB_MISS),
0217     NULL
0218 };
0219
0220 static const struct attribute_group power9_pmu_events_group = {
0221     .name = "events",
0222     .attrs = power9_events_attr,
0223 };
0224
0225 PMU_FORMAT_ATTR(event,      "config:0-51");
0226 PMU_FORMAT_ATTR(pmcxsel,    "config:0-7");
0227 PMU_FORMAT_ATTR(mark,       "config:8");
0228 PMU_FORMAT_ATTR(combine,    "config:10-11");
0229 PMU_FORMAT_ATTR(unit,       "config:12-15");
0230 PMU_FORMAT_ATTR(pmc,        "config:16-19");
0231 PMU_FORMAT_ATTR(cache_sel,  "config:20-23");
0232 PMU_FORMAT_ATTR(sample_mode,    "config:24-28");
0233 PMU_FORMAT_ATTR(thresh_sel, "config:29-31");
0234 PMU_FORMAT_ATTR(thresh_stop,    "config:32-35");
0235 PMU_FORMAT_ATTR(thresh_start,   "config:36-39");
0236 PMU_FORMAT_ATTR(thresh_cmp, "config:40-49");
0237 PMU_FORMAT_ATTR(sdar_mode,  "config:50-51");
0238
0239 static struct attribute *power9_pmu_format_attr[] = {
0240     &format_attr_event.attr,
0241     &format_attr_pmcxsel.attr,
0242     &format_attr_mark.attr,
0243     &format_attr_combine.attr,
0244     &format_attr_unit.attr,
0245     &format_attr_pmc.attr,
0246     &format_attr_cache_sel.attr,
0247     &format_attr_sample_mode.attr,
0248     &format_attr_thresh_sel.attr,
0249     &format_attr_thresh_stop.attr,
0250     &format_attr_thresh_start.attr,
0251     &format_attr_thresh_cmp.attr,
0252     &format_attr_sdar_mode.attr,
0253     NULL,
0254 };
0255
0256 static const struct attribute_group power9_pmu_format_group = {
0257     .name = "format",
0258     .attrs = power9_pmu_format_attr,
0259 };
0260
0261 static struct attribute *power9_pmu_caps_attrs[] = {
0262     NULL
0263 };
0264
0265 static struct attribute_group power9_pmu_caps_group = {
0266     .name  = "caps",
0267     .attrs = power9_pmu_caps_attrs,
0268 };
0269
0270 static const struct attribute_group *power9_pmu_attr_groups[] = {
0271     &power9_pmu_format_group,
0272     &power9_pmu_events_group,
0273     &power9_pmu_caps_group,
0274     NULL,
0275 };
0276
0277 static int power9_generic_events[] = {
0278     [PERF_COUNT_HW_CPU_CYCLES] =            PM_CYC,
0279     [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =   PM_ICT_NOSLOT_CYC,
0280     [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =    PM_CMPLU_STALL,
0281     [PERF_COUNT_HW_INSTRUCTIONS] =          PM_INST_CMPL,
0282     [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] =       PM_BR_CMPL,
0283     [PERF_COUNT_HW_BRANCH_MISSES] =         PM_BR_MPRED_CMPL,
0284     [PERF_COUNT_HW_CACHE_REFERENCES] =      PM_LD_REF_L1,
0285     [PERF_COUNT_HW_CACHE_MISSES] =          PM_LD_MISS_L1_FIN,
0286 };
0287
0288 static u64 power9_bhrb_filter_map(u64 branch_sample_type)
0289 {
0290     u64 pmu_bhrb_filter = 0;
0291
0292     /* BHRB and regular PMU events share the same privilege state
0293      * filter configuration. BHRB is always recorded along with a
0294      * regular PMU event. As the privilege state filter is handled
0295      * in the basic PMC configuration of the accompanying regular
0296      * PMU event, we ignore any separate BHRB specific request.
0297      */
0298
0299     /* No branch filter requested */
0300     if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY)
0301         return pmu_bhrb_filter;
0302
0303     /* Invalid branch filter options - HW does not support */
0304     if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
0305         return -1;
0306
0307     if (branch_sample_type & PERF_SAMPLE_BRANCH_IND_CALL)
0308         return -1;
0309
0310     if (branch_sample_type & PERF_SAMPLE_BRANCH_CALL)
0311         return -1;
0312
0313     if (branch_sample_type & PERF_SAMPLE_BRANCH_ANY_CALL) {
0314         pmu_bhrb_filter |= POWER9_MMCRA_IFM1;
0315         return pmu_bhrb_filter;
0316     }
0317
0318     /* Every thing else is unsupported */
0319     return -1;
0320 }
0321
0322 static void power9_config_bhrb(u64 pmu_bhrb_filter)
0323 {
0324     pmu_bhrb_filter &= POWER9_MMCRA_BHRB_MASK;
0325
0326     /* Enable BHRB filter in PMU */
0327     mtspr(SPRN_MMCRA, (mfspr(SPRN_MMCRA) | pmu_bhrb_filter));
0328 }
0329
0330 #define C(x)    PERF_COUNT_HW_CACHE_##x
0331
0332 /*
0333  * Table of generalized cache-related events.
0334  * 0 means not supported, -1 means nonsensical, other values
0335  * are event codes.
0336  */
0337 static u64 power9_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
0338     [ C(L1D) ] = {
0339         [ C(OP_READ) ] = {
0340             [ C(RESULT_ACCESS) ] = PM_LD_REF_L1,
0341             [ C(RESULT_MISS)   ] = PM_LD_MISS_L1_FIN,
0342         },
0343         [ C(OP_WRITE) ] = {
0344             [ C(RESULT_ACCESS) ] = 0,
0345             [ C(RESULT_MISS)   ] = PM_ST_MISS_L1,
0346         },
0347         [ C(OP_PREFETCH) ] = {
0348             [ C(RESULT_ACCESS) ] = PM_L1_PREF,
0349             [ C(RESULT_MISS)   ] = 0,
0350         },
0351     },
0352     [ C(L1I) ] = {
0353         [ C(OP_READ) ] = {
0354             [ C(RESULT_ACCESS) ] = PM_INST_FROM_L1,
0355             [ C(RESULT_MISS)   ] = PM_L1_ICACHE_MISS,
0356         },
0357         [ C(OP_WRITE) ] = {
0358             [ C(RESULT_ACCESS) ] = PM_L1_DEMAND_WRITE,
0359             [ C(RESULT_MISS)   ] = -1,
0360         },
0361         [ C(OP_PREFETCH) ] = {
0362             [ C(RESULT_ACCESS) ] = PM_IC_PREF_WRITE,
0363             [ C(RESULT_MISS)   ] = 0,
0364         },
0365     },
0366     [ C(LL) ] = {
0367         [ C(OP_READ) ] = {
0368             [ C(RESULT_ACCESS) ] = PM_DATA_FROM_L3,
0369             [ C(RESULT_MISS)   ] = PM_DATA_FROM_L3MISS,
0370         },
0371         [ C(OP_WRITE) ] = {
0372             [ C(RESULT_ACCESS) ] = 0,
0373             [ C(RESULT_MISS)   ] = 0,
0374         },
0375         [ C(OP_PREFETCH) ] = {
0376             [ C(RESULT_ACCESS) ] = PM_L3_PREF_ALL,
0377             [ C(RESULT_MISS)   ] = 0,
0378         },
0379     },
0380     [ C(DTLB) ] = {
0381         [ C(OP_READ) ] = {
0382             [ C(RESULT_ACCESS) ] = 0,
0383             [ C(RESULT_MISS)   ] = PM_DTLB_MISS,
0384         },
0385         [ C(OP_WRITE) ] = {
0386             [ C(RESULT_ACCESS) ] = -1,
0387             [ C(RESULT_MISS)   ] = -1,
0388         },
0389         [ C(OP_PREFETCH) ] = {
0390             [ C(RESULT_ACCESS) ] = -1,
0391             [ C(RESULT_MISS)   ] = -1,
0392         },
0393     },
0394     [ C(ITLB) ] = {
0395         [ C(OP_READ) ] = {
0396             [ C(RESULT_ACCESS) ] = 0,
0397             [ C(RESULT_MISS)   ] = PM_ITLB_MISS,
0398         },
0399         [ C(OP_WRITE) ] = {
0400             [ C(RESULT_ACCESS) ] = -1,
0401             [ C(RESULT_MISS)   ] = -1,
0402         },
0403         [ C(OP_PREFETCH) ] = {
0404             [ C(RESULT_ACCESS) ] = -1,
0405             [ C(RESULT_MISS)   ] = -1,
0406         },
0407     },
0408     [ C(BPU) ] = {
0409         [ C(OP_READ) ] = {
0410             [ C(RESULT_ACCESS) ] = PM_BR_CMPL,
0411             [ C(RESULT_MISS)   ] = PM_BR_MPRED_CMPL,
0412         },
0413         [ C(OP_WRITE) ] = {
0414             [ C(RESULT_ACCESS) ] = -1,
0415             [ C(RESULT_MISS)   ] = -1,
0416         },
0417         [ C(OP_PREFETCH) ] = {
0418             [ C(RESULT_ACCESS) ] = -1,
0419             [ C(RESULT_MISS)   ] = -1,
0420         },
0421     },
0422     [ C(NODE) ] = {
0423         [ C(OP_READ) ] = {
0424             [ C(RESULT_ACCESS) ] = -1,
0425             [ C(RESULT_MISS)   ] = -1,
0426         },
0427         [ C(OP_WRITE) ] = {
0428             [ C(RESULT_ACCESS) ] = -1,
0429             [ C(RESULT_MISS)   ] = -1,
0430         },
0431         [ C(OP_PREFETCH) ] = {
0432             [ C(RESULT_ACCESS) ] = -1,
0433             [ C(RESULT_MISS)   ] = -1,
0434         },
0435     },
0436 };
0437
0438 #undef C
0439
0440 static struct power_pmu power9_pmu = {
0441     .name           = "POWER9",
0442     .n_counter      = MAX_PMU_COUNTERS,
0443     .add_fields     = ISA207_ADD_FIELDS,
0444     .test_adder     = ISA207_TEST_ADDER,
0445     .group_constraint_mask  = CNST_CACHE_PMC4_MASK,
0446     .group_constraint_val   = CNST_CACHE_PMC4_VAL,
0447     .compute_mmcr       = isa207_compute_mmcr,
0448     .config_bhrb        = power9_config_bhrb,
0449     .bhrb_filter_map    = power9_bhrb_filter_map,
0450     .get_constraint     = isa207_get_constraint,
0451     .get_alternatives   = power9_get_alternatives,
0452     .get_mem_data_src   = isa207_get_mem_data_src,
0453     .get_mem_weight     = isa207_get_mem_weight,
0454     .disable_pmc        = isa207_disable_pmc,
0455     .flags          = PPMU_HAS_SIER | PPMU_ARCH_207S,
0456     .n_generic      = ARRAY_SIZE(power9_generic_events),
0457     .generic_events     = power9_generic_events,
0458     .cache_events       = &power9_cache_events,
0459     .attr_groups        = power9_pmu_attr_groups,
0460     .bhrb_nr        = 32,
0461     .capabilities           = PERF_PMU_CAP_EXTENDED_REGS,
0462     .check_attr_config  = power9_check_attr_config,
0463 };
0464
0465 int __init init_power9_pmu(void)
0466 {
0467     int rc = 0;
0468     unsigned int pvr = mfspr(SPRN_PVR);
0469
0470     if (PVR_VER(pvr) != PVR_POWER9)
0471         return -ENODEV;
0472
0473     /* Blacklist events */
0474     if (!(pvr & PVR_POWER9_CUMULUS)) {
0475         if ((PVR_CFG(pvr) == 2) && (PVR_MIN(pvr) == 1)) {
0476             power9_pmu.blacklist_ev = p9_dd21_bl_ev;
0477             power9_pmu.n_blacklist_ev = ARRAY_SIZE(p9_dd21_bl_ev);
0478         } else if ((PVR_CFG(pvr) == 2) && (PVR_MIN(pvr) == 2)) {
0479             power9_pmu.blacklist_ev = p9_dd22_bl_ev;
0480             power9_pmu.n_blacklist_ev = ARRAY_SIZE(p9_dd22_bl_ev);
0481         }
0482     }
0483
0484     /* Set the PERF_REG_EXTENDED_MASK here */
0485     PERF_REG_EXTENDED_MASK = PERF_REG_PMU_MASK_300;
0486
0487     rc = register_power_pmu(&power9_pmu);
0488     if (rc)
0489         return rc;
0490
0491     /* Tell userspace that EBB is supported */
0492     cur_cpu_spec->cpu_user_features2 |= PPC_FEATURE2_EBB;
0493
0494     return 0;
0495 }