Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * RISC-V performance counter support.
0004  *
0005  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
0006  *
0007  * This code is based on ARM perf event code which is in turn based on
0008  * sparc64 and x86 code.
0009  */
0010 
0011 #define pr_fmt(fmt) "riscv-pmu-sbi: " fmt
0012 
0013 #include <linux/mod_devicetable.h>
0014 #include <linux/perf/riscv_pmu.h>
0015 #include <linux/platform_device.h>
0016 #include <linux/irq.h>
0017 #include <linux/irqdomain.h>
0018 #include <linux/of_irq.h>
0019 #include <linux/of.h>
0020 #include <linux/cpu_pm.h>
0021 
0022 #include <asm/sbi.h>
0023 #include <asm/hwcap.h>
0024 
0025 PMU_FORMAT_ATTR(event, "config:0-47");
0026 PMU_FORMAT_ATTR(firmware, "config:63");
0027 
0028 static struct attribute *riscv_arch_formats_attr[] = {
0029     &format_attr_event.attr,
0030     &format_attr_firmware.attr,
0031     NULL,
0032 };
0033 
0034 static struct attribute_group riscv_pmu_format_group = {
0035     .name = "format",
0036     .attrs = riscv_arch_formats_attr,
0037 };
0038 
0039 static const struct attribute_group *riscv_pmu_attr_groups[] = {
0040     &riscv_pmu_format_group,
0041     NULL,
0042 };
0043 
0044 /*
0045  * RISC-V doesn't have hetergenous harts yet. This need to be part of
0046  * per_cpu in case of harts with different pmu counters
0047  */
0048 static union sbi_pmu_ctr_info *pmu_ctr_list;
0049 static unsigned int riscv_pmu_irq;
0050 
0051 struct sbi_pmu_event_data {
0052     union {
0053         union {
0054             struct hw_gen_event {
0055                 uint32_t event_code:16;
0056                 uint32_t event_type:4;
0057                 uint32_t reserved:12;
0058             } hw_gen_event;
0059             struct hw_cache_event {
0060                 uint32_t result_id:1;
0061                 uint32_t op_id:2;
0062                 uint32_t cache_id:13;
0063                 uint32_t event_type:4;
0064                 uint32_t reserved:12;
0065             } hw_cache_event;
0066         };
0067         uint32_t event_idx;
0068     };
0069 };
0070 
0071 static const struct sbi_pmu_event_data pmu_hw_event_map[] = {
0072     [PERF_COUNT_HW_CPU_CYCLES]      = {.hw_gen_event = {
0073                             SBI_PMU_HW_CPU_CYCLES,
0074                             SBI_PMU_EVENT_TYPE_HW, 0}},
0075     [PERF_COUNT_HW_INSTRUCTIONS]        = {.hw_gen_event = {
0076                             SBI_PMU_HW_INSTRUCTIONS,
0077                             SBI_PMU_EVENT_TYPE_HW, 0}},
0078     [PERF_COUNT_HW_CACHE_REFERENCES]    = {.hw_gen_event = {
0079                             SBI_PMU_HW_CACHE_REFERENCES,
0080                             SBI_PMU_EVENT_TYPE_HW, 0}},
0081     [PERF_COUNT_HW_CACHE_MISSES]        = {.hw_gen_event = {
0082                             SBI_PMU_HW_CACHE_MISSES,
0083                             SBI_PMU_EVENT_TYPE_HW, 0}},
0084     [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {.hw_gen_event = {
0085                             SBI_PMU_HW_BRANCH_INSTRUCTIONS,
0086                             SBI_PMU_EVENT_TYPE_HW, 0}},
0087     [PERF_COUNT_HW_BRANCH_MISSES]       = {.hw_gen_event = {
0088                             SBI_PMU_HW_BRANCH_MISSES,
0089                             SBI_PMU_EVENT_TYPE_HW, 0}},
0090     [PERF_COUNT_HW_BUS_CYCLES]      = {.hw_gen_event = {
0091                             SBI_PMU_HW_BUS_CYCLES,
0092                             SBI_PMU_EVENT_TYPE_HW, 0}},
0093     [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = {.hw_gen_event = {
0094                             SBI_PMU_HW_STALLED_CYCLES_FRONTEND,
0095                             SBI_PMU_EVENT_TYPE_HW, 0}},
0096     [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = {.hw_gen_event = {
0097                             SBI_PMU_HW_STALLED_CYCLES_BACKEND,
0098                             SBI_PMU_EVENT_TYPE_HW, 0}},
0099     [PERF_COUNT_HW_REF_CPU_CYCLES]      = {.hw_gen_event = {
0100                             SBI_PMU_HW_REF_CPU_CYCLES,
0101                             SBI_PMU_EVENT_TYPE_HW, 0}},
0102 };
0103 
0104 #define C(x) PERF_COUNT_HW_CACHE_##x
0105 static const struct sbi_pmu_event_data pmu_cache_event_map[PERF_COUNT_HW_CACHE_MAX]
0106 [PERF_COUNT_HW_CACHE_OP_MAX]
0107 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
0108     [C(L1D)] = {
0109         [C(OP_READ)] = {
0110             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0111                     C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0112             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0113                     C(OP_READ), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0114         },
0115         [C(OP_WRITE)] = {
0116             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0117                     C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0118             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0119                     C(OP_WRITE), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0120         },
0121         [C(OP_PREFETCH)] = {
0122             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0123                     C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0124             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0125                     C(OP_PREFETCH), C(L1D), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0126         },
0127     },
0128     [C(L1I)] = {
0129         [C(OP_READ)] = {
0130             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0131                     C(OP_READ), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0132             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS), C(OP_READ),
0133                     C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0134         },
0135         [C(OP_WRITE)] = {
0136             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0137                     C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0138             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0139                     C(OP_WRITE), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0140         },
0141         [C(OP_PREFETCH)] = {
0142             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0143                     C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0144             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0145                     C(OP_PREFETCH), C(L1I), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0146         },
0147     },
0148     [C(LL)] = {
0149         [C(OP_READ)] = {
0150             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0151                     C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0152             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0153                     C(OP_READ), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0154         },
0155         [C(OP_WRITE)] = {
0156             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0157                     C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0158             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0159                     C(OP_WRITE), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0160         },
0161         [C(OP_PREFETCH)] = {
0162             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0163                     C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0164             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0165                     C(OP_PREFETCH), C(LL), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0166         },
0167     },
0168     [C(DTLB)] = {
0169         [C(OP_READ)] = {
0170             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0171                     C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0172             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0173                     C(OP_READ), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0174         },
0175         [C(OP_WRITE)] = {
0176             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0177                     C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0178             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0179                     C(OP_WRITE), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0180         },
0181         [C(OP_PREFETCH)] = {
0182             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0183                     C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0184             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0185                     C(OP_PREFETCH), C(DTLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0186         },
0187     },
0188     [C(ITLB)] = {
0189         [C(OP_READ)] = {
0190             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0191                     C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0192             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0193                     C(OP_READ), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0194         },
0195         [C(OP_WRITE)] = {
0196             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0197                     C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0198             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0199                     C(OP_WRITE), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0200         },
0201         [C(OP_PREFETCH)] = {
0202             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0203                     C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0204             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0205                     C(OP_PREFETCH), C(ITLB), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0206         },
0207     },
0208     [C(BPU)] = {
0209         [C(OP_READ)] = {
0210             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0211                     C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0212             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0213                     C(OP_READ), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0214         },
0215         [C(OP_WRITE)] = {
0216             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0217                     C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0218             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0219                     C(OP_WRITE), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0220         },
0221         [C(OP_PREFETCH)] = {
0222             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0223                     C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0224             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0225                     C(OP_PREFETCH), C(BPU), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0226         },
0227     },
0228     [C(NODE)] = {
0229         [C(OP_READ)] = {
0230             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0231                     C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0232             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0233                     C(OP_READ), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0234         },
0235         [C(OP_WRITE)] = {
0236             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0237                     C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0238             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0239                     C(OP_WRITE), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0240         },
0241         [C(OP_PREFETCH)] = {
0242             [C(RESULT_ACCESS)] = {.hw_cache_event = {C(RESULT_ACCESS),
0243                     C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0244             [C(RESULT_MISS)] = {.hw_cache_event = {C(RESULT_MISS),
0245                     C(OP_PREFETCH), C(NODE), SBI_PMU_EVENT_TYPE_CACHE, 0}},
0246         },
0247     },
0248 };
0249 
0250 static int pmu_sbi_ctr_get_width(int idx)
0251 {
0252     return pmu_ctr_list[idx].width;
0253 }
0254 
0255 static bool pmu_sbi_ctr_is_fw(int cidx)
0256 {
0257     union sbi_pmu_ctr_info *info;
0258 
0259     info = &pmu_ctr_list[cidx];
0260     if (!info)
0261         return false;
0262 
0263     return (info->type == SBI_PMU_CTR_TYPE_FW) ? true : false;
0264 }
0265 
0266 static int pmu_sbi_ctr_get_idx(struct perf_event *event)
0267 {
0268     struct hw_perf_event *hwc = &event->hw;
0269     struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
0270     struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
0271     struct sbiret ret;
0272     int idx;
0273     uint64_t cbase = 0;
0274     uint64_t cmask = GENMASK_ULL(rvpmu->num_counters - 1, 0);
0275     unsigned long cflags = 0;
0276 
0277     if (event->attr.exclude_kernel)
0278         cflags |= SBI_PMU_CFG_FLAG_SET_SINH;
0279     if (event->attr.exclude_user)
0280         cflags |= SBI_PMU_CFG_FLAG_SET_UINH;
0281 
0282     /* retrieve the available counter index */
0283 #if defined(CONFIG_32BIT)
0284     ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, cmask,
0285             cflags, hwc->event_base, hwc->config, hwc->config >> 32);
0286 #else
0287     ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, cmask,
0288             cflags, hwc->event_base, hwc->config, 0);
0289 #endif
0290     if (ret.error) {
0291         pr_debug("Not able to find a counter for event %lx config %llx\n",
0292             hwc->event_base, hwc->config);
0293         return sbi_err_map_linux_errno(ret.error);
0294     }
0295 
0296     idx = ret.value;
0297     if (idx >= rvpmu->num_counters || !pmu_ctr_list[idx].value)
0298         return -ENOENT;
0299 
0300     /* Additional sanity check for the counter id */
0301     if (pmu_sbi_ctr_is_fw(idx)) {
0302         if (!test_and_set_bit(idx, cpuc->used_fw_ctrs))
0303             return idx;
0304     } else {
0305         if (!test_and_set_bit(idx, cpuc->used_hw_ctrs))
0306             return idx;
0307     }
0308 
0309     return -ENOENT;
0310 }
0311 
0312 static void pmu_sbi_ctr_clear_idx(struct perf_event *event)
0313 {
0314 
0315     struct hw_perf_event *hwc = &event->hw;
0316     struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
0317     struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
0318     int idx = hwc->idx;
0319 
0320     if (pmu_sbi_ctr_is_fw(idx))
0321         clear_bit(idx, cpuc->used_fw_ctrs);
0322     else
0323         clear_bit(idx, cpuc->used_hw_ctrs);
0324 }
0325 
0326 static int pmu_event_find_cache(u64 config)
0327 {
0328     unsigned int cache_type, cache_op, cache_result, ret;
0329 
0330     cache_type = (config >>  0) & 0xff;
0331     if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
0332         return -EINVAL;
0333 
0334     cache_op = (config >>  8) & 0xff;
0335     if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
0336         return -EINVAL;
0337 
0338     cache_result = (config >> 16) & 0xff;
0339     if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
0340         return -EINVAL;
0341 
0342     ret = pmu_cache_event_map[cache_type][cache_op][cache_result].event_idx;
0343 
0344     return ret;
0345 }
0346 
0347 static bool pmu_sbi_is_fw_event(struct perf_event *event)
0348 {
0349     u32 type = event->attr.type;
0350     u64 config = event->attr.config;
0351 
0352     if ((type == PERF_TYPE_RAW) && ((config >> 63) == 1))
0353         return true;
0354     else
0355         return false;
0356 }
0357 
0358 static int pmu_sbi_event_map(struct perf_event *event, u64 *econfig)
0359 {
0360     u32 type = event->attr.type;
0361     u64 config = event->attr.config;
0362     int bSoftware;
0363     u64 raw_config_val;
0364     int ret;
0365 
0366     switch (type) {
0367     case PERF_TYPE_HARDWARE:
0368         if (config >= PERF_COUNT_HW_MAX)
0369             return -EINVAL;
0370         ret = pmu_hw_event_map[event->attr.config].event_idx;
0371         break;
0372     case PERF_TYPE_HW_CACHE:
0373         ret = pmu_event_find_cache(config);
0374         break;
0375     case PERF_TYPE_RAW:
0376         /*
0377          * As per SBI specification, the upper 16 bits must be unused for
0378          * a raw event. Use the MSB (63b) to distinguish between hardware
0379          * raw event and firmware events.
0380          */
0381         bSoftware = config >> 63;
0382         raw_config_val = config & RISCV_PMU_RAW_EVENT_MASK;
0383         if (bSoftware) {
0384             if (raw_config_val < SBI_PMU_FW_MAX)
0385                 ret = (raw_config_val & 0xFFFF) |
0386                       (SBI_PMU_EVENT_TYPE_FW << 16);
0387             else
0388                 return -EINVAL;
0389         } else {
0390             ret = RISCV_PMU_RAW_EVENT_IDX;
0391             *econfig = raw_config_val;
0392         }
0393         break;
0394     default:
0395         ret = -EINVAL;
0396         break;
0397     }
0398 
0399     return ret;
0400 }
0401 
0402 static u64 pmu_sbi_ctr_read(struct perf_event *event)
0403 {
0404     struct hw_perf_event *hwc = &event->hw;
0405     int idx = hwc->idx;
0406     struct sbiret ret;
0407     union sbi_pmu_ctr_info info;
0408     u64 val = 0;
0409 
0410     if (pmu_sbi_is_fw_event(event)) {
0411         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_FW_READ,
0412                 hwc->idx, 0, 0, 0, 0, 0);
0413         if (!ret.error)
0414             val = ret.value;
0415     } else {
0416         info = pmu_ctr_list[idx];
0417         val = riscv_pmu_ctr_read_csr(info.csr);
0418         if (IS_ENABLED(CONFIG_32BIT))
0419             val = ((u64)riscv_pmu_ctr_read_csr(info.csr + 0x80)) << 31 | val;
0420     }
0421 
0422     return val;
0423 }
0424 
0425 static void pmu_sbi_ctr_start(struct perf_event *event, u64 ival)
0426 {
0427     struct sbiret ret;
0428     struct hw_perf_event *hwc = &event->hw;
0429     unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
0430 
0431 #if defined(CONFIG_32BIT)
0432     ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx,
0433             1, flag, ival, ival >> 32, 0);
0434 #else
0435     ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, hwc->idx,
0436             1, flag, ival, 0, 0);
0437 #endif
0438     if (ret.error && (ret.error != SBI_ERR_ALREADY_STARTED))
0439         pr_err("Starting counter idx %d failed with error %d\n",
0440             hwc->idx, sbi_err_map_linux_errno(ret.error));
0441 }
0442 
0443 static void pmu_sbi_ctr_stop(struct perf_event *event, unsigned long flag)
0444 {
0445     struct sbiret ret;
0446     struct hw_perf_event *hwc = &event->hw;
0447 
0448     ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, hwc->idx, 1, flag, 0, 0, 0);
0449     if (ret.error && (ret.error != SBI_ERR_ALREADY_STOPPED) &&
0450         flag != SBI_PMU_STOP_FLAG_RESET)
0451         pr_err("Stopping counter idx %d failed with error %d\n",
0452             hwc->idx, sbi_err_map_linux_errno(ret.error));
0453 }
0454 
0455 static int pmu_sbi_find_num_ctrs(void)
0456 {
0457     struct sbiret ret;
0458 
0459     ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_NUM_COUNTERS, 0, 0, 0, 0, 0, 0);
0460     if (!ret.error)
0461         return ret.value;
0462     else
0463         return sbi_err_map_linux_errno(ret.error);
0464 }
0465 
0466 static int pmu_sbi_get_ctrinfo(int nctr)
0467 {
0468     struct sbiret ret;
0469     int i, num_hw_ctr = 0, num_fw_ctr = 0;
0470     union sbi_pmu_ctr_info cinfo;
0471 
0472     pmu_ctr_list = kcalloc(nctr, sizeof(*pmu_ctr_list), GFP_KERNEL);
0473     if (!pmu_ctr_list)
0474         return -ENOMEM;
0475 
0476     for (i = 0; i < nctr; i++) {
0477         ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_GET_INFO, i, 0, 0, 0, 0, 0);
0478         if (ret.error)
0479             /* The logical counter ids are not expected to be contiguous */
0480             continue;
0481         cinfo.value = ret.value;
0482         if (cinfo.type == SBI_PMU_CTR_TYPE_FW)
0483             num_fw_ctr++;
0484         else
0485             num_hw_ctr++;
0486         pmu_ctr_list[i].value = cinfo.value;
0487     }
0488 
0489     pr_info("%d firmware and %d hardware counters\n", num_fw_ctr, num_hw_ctr);
0490 
0491     return 0;
0492 }
0493 
0494 static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu)
0495 {
0496     /*
0497      * No need to check the error because we are disabling all the counters
0498      * which may include counters that are not enabled yet.
0499      */
0500     sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP,
0501           0, GENMASK_ULL(pmu->num_counters - 1, 0), 0, 0, 0, 0);
0502 }
0503 
0504 static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu)
0505 {
0506     struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
0507 
0508     /* No need to check the error here as we can't do anything about the error */
0509     sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, 0,
0510           cpu_hw_evt->used_hw_ctrs[0], 0, 0, 0, 0);
0511 }
0512 
0513 /*
0514  * This function starts all the used counters in two step approach.
0515  * Any counter that did not overflow can be start in a single step
0516  * while the overflowed counters need to be started with updated initialization
0517  * value.
0518  */
0519 static inline void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu,
0520                            unsigned long ctr_ovf_mask)
0521 {
0522     int idx = 0;
0523     struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
0524     struct perf_event *event;
0525     unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE;
0526     unsigned long ctr_start_mask = 0;
0527     uint64_t max_period;
0528     struct hw_perf_event *hwc;
0529     u64 init_val = 0;
0530 
0531     ctr_start_mask = cpu_hw_evt->used_hw_ctrs[0] & ~ctr_ovf_mask;
0532 
0533     /* Start all the counters that did not overflow in a single shot */
0534     sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, 0, ctr_start_mask,
0535           0, 0, 0, 0);
0536 
0537     /* Reinitialize and start all the counter that overflowed */
0538     while (ctr_ovf_mask) {
0539         if (ctr_ovf_mask & 0x01) {
0540             event = cpu_hw_evt->events[idx];
0541             hwc = &event->hw;
0542             max_period = riscv_pmu_ctr_get_width_mask(event);
0543             init_val = local64_read(&hwc->prev_count) & max_period;
0544 #if defined(CONFIG_32BIT)
0545             sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1,
0546                   flag, init_val, init_val >> 32, 0);
0547 #else
0548             sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1,
0549                   flag, init_val, 0, 0);
0550 #endif
0551             perf_event_update_userpage(event);
0552         }
0553         ctr_ovf_mask = ctr_ovf_mask >> 1;
0554         idx++;
0555     }
0556 }
0557 
0558 static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev)
0559 {
0560     struct perf_sample_data data;
0561     struct pt_regs *regs;
0562     struct hw_perf_event *hw_evt;
0563     union sbi_pmu_ctr_info *info;
0564     int lidx, hidx, fidx;
0565     struct riscv_pmu *pmu;
0566     struct perf_event *event;
0567     unsigned long overflow;
0568     unsigned long overflowed_ctrs = 0;
0569     struct cpu_hw_events *cpu_hw_evt = dev;
0570 
0571     if (WARN_ON_ONCE(!cpu_hw_evt))
0572         return IRQ_NONE;
0573 
0574     /* Firmware counter don't support overflow yet */
0575     fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS);
0576     event = cpu_hw_evt->events[fidx];
0577     if (!event) {
0578         csr_clear(CSR_SIP, SIP_LCOFIP);
0579         return IRQ_NONE;
0580     }
0581 
0582     pmu = to_riscv_pmu(event->pmu);
0583     pmu_sbi_stop_hw_ctrs(pmu);
0584 
0585     /* Overflow status register should only be read after counter are stopped */
0586     overflow = csr_read(CSR_SSCOUNTOVF);
0587 
0588     /*
0589      * Overflow interrupt pending bit should only be cleared after stopping
0590      * all the counters to avoid any race condition.
0591      */
0592     csr_clear(CSR_SIP, SIP_LCOFIP);
0593 
0594     /* No overflow bit is set */
0595     if (!overflow)
0596         return IRQ_NONE;
0597 
0598     regs = get_irq_regs();
0599 
0600     for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) {
0601         struct perf_event *event = cpu_hw_evt->events[lidx];
0602 
0603         /* Skip if invalid event or user did not request a sampling */
0604         if (!event || !is_sampling_event(event))
0605             continue;
0606 
0607         info = &pmu_ctr_list[lidx];
0608         /* Do a sanity check */
0609         if (!info || info->type != SBI_PMU_CTR_TYPE_HW)
0610             continue;
0611 
0612         /* compute hardware counter index */
0613         hidx = info->csr - CSR_CYCLE;
0614         /* check if the corresponding bit is set in sscountovf */
0615         if (!(overflow & (1 << hidx)))
0616             continue;
0617 
0618         /*
0619          * Keep a track of overflowed counters so that they can be started
0620          * with updated initial value.
0621          */
0622         overflowed_ctrs |= 1 << lidx;
0623         hw_evt = &event->hw;
0624         riscv_pmu_event_update(event);
0625         perf_sample_data_init(&data, 0, hw_evt->last_period);
0626         if (riscv_pmu_event_set_period(event)) {
0627             /*
0628              * Unlike other ISAs, RISC-V don't have to disable interrupts
0629              * to avoid throttling here. As per the specification, the
0630              * interrupt remains disabled until the OF bit is set.
0631              * Interrupts are enabled again only during the start.
0632              * TODO: We will need to stop the guest counters once
0633              * virtualization support is added.
0634              */
0635             perf_event_overflow(event, &data, regs);
0636         }
0637     }
0638     pmu_sbi_start_overflow_mask(pmu, overflowed_ctrs);
0639 
0640     return IRQ_HANDLED;
0641 }
0642 
0643 static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node)
0644 {
0645     struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node);
0646     struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events);
0647 
0648     /* Enable the access for TIME csr only from the user mode now */
0649     csr_write(CSR_SCOUNTEREN, 0x2);
0650 
0651     /* Stop all the counters so that they can be enabled from perf */
0652     pmu_sbi_stop_all(pmu);
0653 
0654     if (riscv_isa_extension_available(NULL, SSCOFPMF)) {
0655         cpu_hw_evt->irq = riscv_pmu_irq;
0656         csr_clear(CSR_IP, BIT(RV_IRQ_PMU));
0657         csr_set(CSR_IE, BIT(RV_IRQ_PMU));
0658         enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE);
0659     }
0660 
0661     return 0;
0662 }
0663 
0664 static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node)
0665 {
0666     if (riscv_isa_extension_available(NULL, SSCOFPMF)) {
0667         disable_percpu_irq(riscv_pmu_irq);
0668         csr_clear(CSR_IE, BIT(RV_IRQ_PMU));
0669     }
0670 
0671     /* Disable all counters access for user mode now */
0672     csr_write(CSR_SCOUNTEREN, 0x0);
0673 
0674     return 0;
0675 }
0676 
0677 static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev)
0678 {
0679     int ret;
0680     struct cpu_hw_events __percpu *hw_events = pmu->hw_events;
0681     struct device_node *cpu, *child;
0682     struct irq_domain *domain = NULL;
0683 
0684     if (!riscv_isa_extension_available(NULL, SSCOFPMF))
0685         return -EOPNOTSUPP;
0686 
0687     for_each_of_cpu_node(cpu) {
0688         child = of_get_compatible_child(cpu, "riscv,cpu-intc");
0689         if (!child) {
0690             pr_err("Failed to find INTC node\n");
0691             of_node_put(cpu);
0692             return -ENODEV;
0693         }
0694         domain = irq_find_host(child);
0695         of_node_put(child);
0696         if (domain) {
0697             of_node_put(cpu);
0698             break;
0699         }
0700     }
0701     if (!domain) {
0702         pr_err("Failed to find INTC IRQ root domain\n");
0703         return -ENODEV;
0704     }
0705 
0706     riscv_pmu_irq = irq_create_mapping(domain, RV_IRQ_PMU);
0707     if (!riscv_pmu_irq) {
0708         pr_err("Failed to map PMU interrupt for node\n");
0709         return -ENODEV;
0710     }
0711 
0712     ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events);
0713     if (ret) {
0714         pr_err("registering percpu irq failed [%d]\n", ret);
0715         return ret;
0716     }
0717 
0718     return 0;
0719 }
0720 
0721 #ifdef CONFIG_CPU_PM
0722 static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd,
0723                 void *v)
0724 {
0725     struct riscv_pmu *rvpmu = container_of(b, struct riscv_pmu, riscv_pm_nb);
0726     struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
0727     int enabled = bitmap_weight(cpuc->used_hw_ctrs, RISCV_MAX_COUNTERS);
0728     struct perf_event *event;
0729     int idx;
0730 
0731     if (!enabled)
0732         return NOTIFY_OK;
0733 
0734     for (idx = 0; idx < RISCV_MAX_COUNTERS; idx++) {
0735         event = cpuc->events[idx];
0736         if (!event)
0737             continue;
0738 
0739         switch (cmd) {
0740         case CPU_PM_ENTER:
0741             /*
0742              * Stop and update the counter
0743              */
0744             riscv_pmu_stop(event, PERF_EF_UPDATE);
0745             break;
0746         case CPU_PM_EXIT:
0747         case CPU_PM_ENTER_FAILED:
0748             /*
0749              * Restore and enable the counter.
0750              *
0751              * Requires RCU read locking to be functional,
0752              * wrap the call within RCU_NONIDLE to make the
0753              * RCU subsystem aware this cpu is not idle from
0754              * an RCU perspective for the riscv_pmu_start() call
0755              * duration.
0756              */
0757             RCU_NONIDLE(riscv_pmu_start(event, PERF_EF_RELOAD));
0758             break;
0759         default:
0760             break;
0761         }
0762     }
0763 
0764     return NOTIFY_OK;
0765 }
0766 
0767 static int riscv_pm_pmu_register(struct riscv_pmu *pmu)
0768 {
0769     pmu->riscv_pm_nb.notifier_call = riscv_pm_pmu_notify;
0770     return cpu_pm_register_notifier(&pmu->riscv_pm_nb);
0771 }
0772 
0773 static void riscv_pm_pmu_unregister(struct riscv_pmu *pmu)
0774 {
0775     cpu_pm_unregister_notifier(&pmu->riscv_pm_nb);
0776 }
0777 #else
0778 static inline int riscv_pm_pmu_register(struct riscv_pmu *pmu) { return 0; }
0779 static inline void riscv_pm_pmu_unregister(struct riscv_pmu *pmu) { }
0780 #endif
0781 
0782 static void riscv_pmu_destroy(struct riscv_pmu *pmu)
0783 {
0784     riscv_pm_pmu_unregister(pmu);
0785     cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
0786 }
0787 
0788 static int pmu_sbi_device_probe(struct platform_device *pdev)
0789 {
0790     struct riscv_pmu *pmu = NULL;
0791     int num_counters;
0792     int ret = -ENODEV;
0793 
0794     pr_info("SBI PMU extension is available\n");
0795     pmu = riscv_pmu_alloc();
0796     if (!pmu)
0797         return -ENOMEM;
0798 
0799     num_counters = pmu_sbi_find_num_ctrs();
0800     if (num_counters < 0) {
0801         pr_err("SBI PMU extension doesn't provide any counters\n");
0802         goto out_free;
0803     }
0804 
0805     /* cache all the information about counters now */
0806     if (pmu_sbi_get_ctrinfo(num_counters))
0807         goto out_free;
0808 
0809     ret = pmu_sbi_setup_irqs(pmu, pdev);
0810     if (ret < 0) {
0811         pr_info("Perf sampling/filtering is not supported as sscof extension is not available\n");
0812         pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
0813         pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE;
0814     }
0815     pmu->pmu.attr_groups = riscv_pmu_attr_groups;
0816     pmu->num_counters = num_counters;
0817     pmu->ctr_start = pmu_sbi_ctr_start;
0818     pmu->ctr_stop = pmu_sbi_ctr_stop;
0819     pmu->event_map = pmu_sbi_event_map;
0820     pmu->ctr_get_idx = pmu_sbi_ctr_get_idx;
0821     pmu->ctr_get_width = pmu_sbi_ctr_get_width;
0822     pmu->ctr_clear_idx = pmu_sbi_ctr_clear_idx;
0823     pmu->ctr_read = pmu_sbi_ctr_read;
0824 
0825     ret = cpuhp_state_add_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node);
0826     if (ret)
0827         return ret;
0828 
0829     ret = riscv_pm_pmu_register(pmu);
0830     if (ret)
0831         goto out_unregister;
0832 
0833     ret = perf_pmu_register(&pmu->pmu, "cpu", PERF_TYPE_RAW);
0834     if (ret)
0835         goto out_unregister;
0836 
0837     return 0;
0838 
0839 out_unregister:
0840     riscv_pmu_destroy(pmu);
0841 
0842 out_free:
0843     kfree(pmu);
0844     return ret;
0845 }
0846 
0847 static struct platform_driver pmu_sbi_driver = {
0848     .probe      = pmu_sbi_device_probe,
0849     .driver     = {
0850         .name   = RISCV_PMU_PDEV_NAME,
0851     },
0852 };
0853 
0854 static int __init pmu_sbi_devinit(void)
0855 {
0856     int ret;
0857     struct platform_device *pdev;
0858 
0859     if (sbi_spec_version < sbi_mk_version(0, 3) ||
0860         sbi_probe_extension(SBI_EXT_PMU) <= 0) {
0861         return 0;
0862     }
0863 
0864     ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_RISCV_STARTING,
0865                       "perf/riscv/pmu:starting",
0866                       pmu_sbi_starting_cpu, pmu_sbi_dying_cpu);
0867     if (ret) {
0868         pr_err("CPU hotplug notifier could not be registered: %d\n",
0869                ret);
0870         return ret;
0871     }
0872 
0873     ret = platform_driver_register(&pmu_sbi_driver);
0874     if (ret)
0875         return ret;
0876 
0877     pdev = platform_device_register_simple(RISCV_PMU_PDEV_NAME, -1, NULL, 0);
0878     if (IS_ERR(pdev)) {
0879         platform_driver_unregister(&pmu_sbi_driver);
0880         return PTR_ERR(pdev);
0881     }
0882 
0883     /* Notify legacy implementation that SBI pmu is available*/
0884     riscv_pmu_legacy_skip_init();
0885 
0886     return ret;
0887 }
0888 device_initcall(pmu_sbi_devinit)
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!