drivers/perf/riscv_pmu.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * RISC-V performance counter support.
0004  *
0005  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
0006  *
0007  * This implementation is based on old RISC-V perf and ARM perf event code
0008  * which are in turn based on sparc64 and x86 code.
0009  */
0010
0011 #include <linux/cpumask.h>
0012 #include <linux/irq.h>
0013 #include <linux/irqdesc.h>
0014 #include <linux/perf/riscv_pmu.h>
0015 #include <linux/printk.h>
0016 #include <linux/smp.h>
0017
0018 #include <asm/sbi.h>
0019
0020 static unsigned long csr_read_num(int csr_num)
0021 {
0022 #define switchcase_csr_read(__csr_num, __val)       {\
0023     case __csr_num:                 \
0024         __val = csr_read(__csr_num);        \
0025         break; }
0026 #define switchcase_csr_read_2(__csr_num, __val)     {\
0027     switchcase_csr_read(__csr_num + 0, __val)    \
0028     switchcase_csr_read(__csr_num + 1, __val)}
0029 #define switchcase_csr_read_4(__csr_num, __val)     {\
0030     switchcase_csr_read_2(__csr_num + 0, __val)  \
0031     switchcase_csr_read_2(__csr_num + 2, __val)}
0032 #define switchcase_csr_read_8(__csr_num, __val)     {\
0033     switchcase_csr_read_4(__csr_num + 0, __val)  \
0034     switchcase_csr_read_4(__csr_num + 4, __val)}
0035 #define switchcase_csr_read_16(__csr_num, __val)    {\
0036     switchcase_csr_read_8(__csr_num + 0, __val)  \
0037     switchcase_csr_read_8(__csr_num + 8, __val)}
0038 #define switchcase_csr_read_32(__csr_num, __val)    {\
0039     switchcase_csr_read_16(__csr_num + 0, __val)     \
0040     switchcase_csr_read_16(__csr_num + 16, __val)}
0041
0042     unsigned long ret = 0;
0043
0044     switch (csr_num) {
0045     switchcase_csr_read_32(CSR_CYCLE, ret)
0046     switchcase_csr_read_32(CSR_CYCLEH, ret)
0047     default :
0048         break;
0049     }
0050
0051     return ret;
0052 #undef switchcase_csr_read_32
0053 #undef switchcase_csr_read_16
0054 #undef switchcase_csr_read_8
0055 #undef switchcase_csr_read_4
0056 #undef switchcase_csr_read_2
0057 #undef switchcase_csr_read
0058 }
0059
0060 /*
0061  * Read the CSR of a corresponding counter.
0062  */
0063 unsigned long riscv_pmu_ctr_read_csr(unsigned long csr)
0064 {
0065     if (csr < CSR_CYCLE || csr > CSR_HPMCOUNTER31H ||
0066        (csr > CSR_HPMCOUNTER31 && csr < CSR_CYCLEH)) {
0067         pr_err("Invalid performance counter csr %lx\n", csr);
0068         return -EINVAL;
0069     }
0070
0071     return csr_read_num(csr);
0072 }
0073
0074 u64 riscv_pmu_ctr_get_width_mask(struct perf_event *event)
0075 {
0076     int cwidth;
0077     struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
0078     struct hw_perf_event *hwc = &event->hw;
0079
0080     if (!rvpmu->ctr_get_width)
0081     /**
0082      * If the pmu driver doesn't support counter width, set it to default
0083      * maximum allowed by the specification.
0084      */
0085         cwidth = 63;
0086     else {
0087         if (hwc->idx == -1)
0088             /* Handle init case where idx is not initialized yet */
0089             cwidth = rvpmu->ctr_get_width(0);
0090         else
0091             cwidth = rvpmu->ctr_get_width(hwc->idx);
0092     }
0093
0094     return GENMASK_ULL(cwidth, 0);
0095 }
0096
0097 u64 riscv_pmu_event_update(struct perf_event *event)
0098 {
0099     struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
0100     struct hw_perf_event *hwc = &event->hw;
0101     u64 prev_raw_count, new_raw_count;
0102     unsigned long cmask;
0103     u64 oldval, delta;
0104
0105     if (!rvpmu->ctr_read)
0106         return 0;
0107
0108     cmask = riscv_pmu_ctr_get_width_mask(event);
0109
0110     do {
0111         prev_raw_count = local64_read(&hwc->prev_count);
0112         new_raw_count = rvpmu->ctr_read(event);
0113         oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count,
0114                      new_raw_count);
0115     } while (oldval != prev_raw_count);
0116
0117     delta = (new_raw_count - prev_raw_count) & cmask;
0118     local64_add(delta, &event->count);
0119     local64_sub(delta, &hwc->period_left);
0120
0121     return delta;
0122 }
0123
0124 void riscv_pmu_stop(struct perf_event *event, int flags)
0125 {
0126     struct hw_perf_event *hwc = &event->hw;
0127     struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
0128
0129     WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
0130
0131     if (!(hwc->state & PERF_HES_STOPPED)) {
0132         if (rvpmu->ctr_stop) {
0133             rvpmu->ctr_stop(event, 0);
0134             hwc->state |= PERF_HES_STOPPED;
0135         }
0136         riscv_pmu_event_update(event);
0137         hwc->state |= PERF_HES_UPTODATE;
0138     }
0139 }
0140
0141 int riscv_pmu_event_set_period(struct perf_event *event)
0142 {
0143     struct hw_perf_event *hwc = &event->hw;
0144     s64 left = local64_read(&hwc->period_left);
0145     s64 period = hwc->sample_period;
0146     int overflow = 0;
0147     uint64_t max_period = riscv_pmu_ctr_get_width_mask(event);
0148
0149     if (unlikely(left <= -period)) {
0150         left = period;
0151         local64_set(&hwc->period_left, left);
0152         hwc->last_period = period;
0153         overflow = 1;
0154     }
0155
0156     if (unlikely(left <= 0)) {
0157         left += period;
0158         local64_set(&hwc->period_left, left);
0159         hwc->last_period = period;
0160         overflow = 1;
0161     }
0162
0163     /*
0164      * Limit the maximum period to prevent the counter value
0165      * from overtaking the one we are about to program. In
0166      * effect we are reducing max_period to account for
0167      * interrupt latency (and we are being very conservative).
0168      */
0169     if (left > (max_period >> 1))
0170         left = (max_period >> 1);
0171
0172     local64_set(&hwc->prev_count, (u64)-left);
0173
0174     return overflow;
0175 }
0176
0177 void riscv_pmu_start(struct perf_event *event, int flags)
0178 {
0179     struct hw_perf_event *hwc = &event->hw;
0180     struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
0181     uint64_t max_period = riscv_pmu_ctr_get_width_mask(event);
0182     u64 init_val;
0183
0184     if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
0185         return;
0186
0187     if (flags & PERF_EF_RELOAD)
0188         WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
0189
0190     hwc->state = 0;
0191     riscv_pmu_event_set_period(event);
0192     init_val = local64_read(&hwc->prev_count) & max_period;
0193     rvpmu->ctr_start(event, init_val);
0194     perf_event_update_userpage(event);
0195 }
0196
0197 static int riscv_pmu_add(struct perf_event *event, int flags)
0198 {
0199     struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
0200     struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
0201     struct hw_perf_event *hwc = &event->hw;
0202     int idx;
0203
0204     idx = rvpmu->ctr_get_idx(event);
0205     if (idx < 0)
0206         return idx;
0207
0208     hwc->idx = idx;
0209     cpuc->events[idx] = event;
0210     cpuc->n_events++;
0211     hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
0212     if (flags & PERF_EF_START)
0213         riscv_pmu_start(event, PERF_EF_RELOAD);
0214
0215     /* Propagate our changes to the userspace mapping. */
0216     perf_event_update_userpage(event);
0217
0218     return 0;
0219 }
0220
0221 static void riscv_pmu_del(struct perf_event *event, int flags)
0222 {
0223     struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
0224     struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events);
0225     struct hw_perf_event *hwc = &event->hw;
0226
0227     riscv_pmu_stop(event, PERF_EF_UPDATE);
0228     cpuc->events[hwc->idx] = NULL;
0229     /* The firmware need to reset the counter mapping */
0230     if (rvpmu->ctr_stop)
0231         rvpmu->ctr_stop(event, RISCV_PMU_STOP_FLAG_RESET);
0232     cpuc->n_events--;
0233     if (rvpmu->ctr_clear_idx)
0234         rvpmu->ctr_clear_idx(event);
0235     perf_event_update_userpage(event);
0236     hwc->idx = -1;
0237 }
0238
0239 static void riscv_pmu_read(struct perf_event *event)
0240 {
0241     riscv_pmu_event_update(event);
0242 }
0243
0244 static int riscv_pmu_event_init(struct perf_event *event)
0245 {
0246     struct hw_perf_event *hwc = &event->hw;
0247     struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu);
0248     int mapped_event;
0249     u64 event_config = 0;
0250     uint64_t cmask;
0251
0252     hwc->flags = 0;
0253     mapped_event = rvpmu->event_map(event, &event_config);
0254     if (mapped_event < 0) {
0255         pr_debug("event %x:%llx not supported\n", event->attr.type,
0256              event->attr.config);
0257         return mapped_event;
0258     }
0259
0260     /*
0261      * idx is set to -1 because the index of a general event should not be
0262      * decided until binding to some counter in pmu->add().
0263      * config will contain the information about counter CSR
0264      * the idx will contain the counter index
0265      */
0266     hwc->config = event_config;
0267     hwc->idx = -1;
0268     hwc->event_base = mapped_event;
0269
0270     if (!is_sampling_event(event)) {
0271         /*
0272          * For non-sampling runs, limit the sample_period to half
0273          * of the counter width. That way, the new counter value
0274          * is far less likely to overtake the previous one unless
0275          * you have some serious IRQ latency issues.
0276          */
0277         cmask = riscv_pmu_ctr_get_width_mask(event);
0278         hwc->sample_period  =  cmask >> 1;
0279         hwc->last_period    = hwc->sample_period;
0280         local64_set(&hwc->period_left, hwc->sample_period);
0281     }
0282
0283     return 0;
0284 }
0285
0286 struct riscv_pmu *riscv_pmu_alloc(void)
0287 {
0288     struct riscv_pmu *pmu;
0289     int cpuid, i;
0290     struct cpu_hw_events *cpuc;
0291
0292     pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
0293     if (!pmu)
0294         goto out;
0295
0296     pmu->hw_events = alloc_percpu_gfp(struct cpu_hw_events, GFP_KERNEL);
0297     if (!pmu->hw_events) {
0298         pr_info("failed to allocate per-cpu PMU data.\n");
0299         goto out_free_pmu;
0300     }
0301
0302     for_each_possible_cpu(cpuid) {
0303         cpuc = per_cpu_ptr(pmu->hw_events, cpuid);
0304         cpuc->n_events = 0;
0305         for (i = 0; i < RISCV_MAX_COUNTERS; i++)
0306             cpuc->events[i] = NULL;
0307     }
0308     pmu->pmu = (struct pmu) {
0309         .event_init = riscv_pmu_event_init,
0310         .add        = riscv_pmu_add,
0311         .del        = riscv_pmu_del,
0312         .start      = riscv_pmu_start,
0313         .stop       = riscv_pmu_stop,
0314         .read       = riscv_pmu_read,
0315     };
0316
0317     return pmu;
0318
0319 out_free_pmu:
0320     kfree(pmu);
0321 out:
0322     return NULL;
0323 }