Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * ARMv8 PMUv3 Performance Events handling code.
0004  *
0005  * Copyright (C) 2012 ARM Limited
0006  * Author: Will Deacon <will.deacon@arm.com>
0007  *
0008  * This code is based heavily on the ARMv7 perf event code.
0009  */
0010 
0011 #include <asm/irq_regs.h>
0012 #include <asm/perf_event.h>
0013 #include <asm/sysreg.h>
0014 #include <asm/virt.h>
0015 
0016 #include <clocksource/arm_arch_timer.h>
0017 
0018 #include <linux/acpi.h>
0019 #include <linux/clocksource.h>
0020 #include <linux/kvm_host.h>
0021 #include <linux/of.h>
0022 #include <linux/perf/arm_pmu.h>
0023 #include <linux/platform_device.h>
0024 #include <linux/sched_clock.h>
0025 #include <linux/smp.h>
0026 
0027 /* ARMv8 Cortex-A53 specific event types. */
0028 #define ARMV8_A53_PERFCTR_PREF_LINEFILL             0xC2
0029 
0030 /* ARMv8 Cavium ThunderX specific event types. */
0031 #define ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST         0xE9
0032 #define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS     0xEA
0033 #define ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS       0xEB
0034 #define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS     0xEC
0035 #define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS       0xED
0036 
0037 /*
0038  * ARMv8 Architectural defined events, not all of these may
0039  * be supported on any given implementation. Unsupported events will
0040  * be disabled at run-time based on the PMCEID registers.
0041  */
0042 static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
0043     PERF_MAP_ALL_UNSUPPORTED,
0044     [PERF_COUNT_HW_CPU_CYCLES]      = ARMV8_PMUV3_PERFCTR_CPU_CYCLES,
0045     [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV8_PMUV3_PERFCTR_INST_RETIRED,
0046     [PERF_COUNT_HW_CACHE_REFERENCES]    = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
0047     [PERF_COUNT_HW_CACHE_MISSES]        = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
0048     [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED,
0049     [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
0050     [PERF_COUNT_HW_BUS_CYCLES]      = ARMV8_PMUV3_PERFCTR_BUS_CYCLES,
0051     [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV8_PMUV3_PERFCTR_STALL_FRONTEND,
0052     [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV8_PMUV3_PERFCTR_STALL_BACKEND,
0053 };
0054 
0055 static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
0056                         [PERF_COUNT_HW_CACHE_OP_MAX]
0057                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
0058     PERF_CACHE_MAP_ALL_UNSUPPORTED,
0059 
0060     [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
0061     [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
0062 
0063     [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
0064     [C(L1I)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
0065 
0066     [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL,
0067     [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_TLB,
0068 
0069     [C(ITLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL,
0070     [C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_TLB,
0071 
0072     [C(LL)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD,
0073     [C(LL)][C(OP_READ)][C(RESULT_ACCESS)]   = ARMV8_PMUV3_PERFCTR_LL_CACHE_RD,
0074 
0075     [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_PMUV3_PERFCTR_BR_PRED,
0076     [C(BPU)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
0077 };
0078 
0079 static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
0080                           [PERF_COUNT_HW_CACHE_OP_MAX]
0081                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
0082     PERF_CACHE_MAP_ALL_UNSUPPORTED,
0083 
0084     [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_A53_PERFCTR_PREF_LINEFILL,
0085 
0086     [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
0087     [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
0088 };
0089 
0090 static const unsigned armv8_a57_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
0091                           [PERF_COUNT_HW_CACHE_OP_MAX]
0092                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
0093     PERF_CACHE_MAP_ALL_UNSUPPORTED,
0094 
0095     [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
0096     [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD,
0097     [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
0098     [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
0099 
0100     [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
0101     [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
0102 
0103     [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
0104     [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
0105 };
0106 
0107 static const unsigned armv8_a73_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
0108                           [PERF_COUNT_HW_CACHE_OP_MAX]
0109                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
0110     PERF_CACHE_MAP_ALL_UNSUPPORTED,
0111 
0112     [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
0113     [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
0114 };
0115 
0116 static const unsigned armv8_thunder_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
0117                            [PERF_COUNT_HW_CACHE_OP_MAX]
0118                            [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
0119     PERF_CACHE_MAP_ALL_UNSUPPORTED,
0120 
0121     [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
0122     [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD,
0123     [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
0124     [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_THUNDER_PERFCTR_L1D_CACHE_MISS_ST,
0125     [C(L1D)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_ACCESS,
0126     [C(L1D)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1D_CACHE_PREF_MISS,
0127 
0128     [C(L1I)][C(OP_PREFETCH)][C(RESULT_ACCESS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS,
0129     [C(L1I)][C(OP_PREFETCH)][C(RESULT_MISS)] = ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS,
0130 
0131     [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD,
0132     [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
0133     [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
0134     [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
0135 };
0136 
0137 static const unsigned armv8_vulcan_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
0138                           [PERF_COUNT_HW_CACHE_OP_MAX]
0139                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
0140     PERF_CACHE_MAP_ALL_UNSUPPORTED,
0141 
0142     [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)]  = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD,
0143     [C(L1D)][C(OP_READ)][C(RESULT_MISS)]    = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD,
0144     [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR,
0145     [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)]   = ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR,
0146 
0147     [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD,
0148     [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR,
0149     [C(DTLB)][C(OP_READ)][C(RESULT_MISS)]   = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD,
0150     [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)]  = ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR,
0151 
0152     [C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD,
0153     [C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR,
0154 };
0155 
0156 static ssize_t
0157 armv8pmu_events_sysfs_show(struct device *dev,
0158                struct device_attribute *attr, char *page)
0159 {
0160     struct perf_pmu_events_attr *pmu_attr;
0161 
0162     pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
0163 
0164     return sprintf(page, "event=0x%04llx\n", pmu_attr->id);
0165 }
0166 
0167 #define ARMV8_EVENT_ATTR(name, config)                      \
0168     PMU_EVENT_ATTR_ID(name, armv8pmu_events_sysfs_show, config)
0169 
0170 static struct attribute *armv8_pmuv3_event_attrs[] = {
0171     ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR),
0172     ARMV8_EVENT_ATTR(l1i_cache_refill, ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL),
0173     ARMV8_EVENT_ATTR(l1i_tlb_refill, ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL),
0174     ARMV8_EVENT_ATTR(l1d_cache_refill, ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL),
0175     ARMV8_EVENT_ATTR(l1d_cache, ARMV8_PMUV3_PERFCTR_L1D_CACHE),
0176     ARMV8_EVENT_ATTR(l1d_tlb_refill, ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL),
0177     ARMV8_EVENT_ATTR(ld_retired, ARMV8_PMUV3_PERFCTR_LD_RETIRED),
0178     ARMV8_EVENT_ATTR(st_retired, ARMV8_PMUV3_PERFCTR_ST_RETIRED),
0179     ARMV8_EVENT_ATTR(inst_retired, ARMV8_PMUV3_PERFCTR_INST_RETIRED),
0180     ARMV8_EVENT_ATTR(exc_taken, ARMV8_PMUV3_PERFCTR_EXC_TAKEN),
0181     ARMV8_EVENT_ATTR(exc_return, ARMV8_PMUV3_PERFCTR_EXC_RETURN),
0182     ARMV8_EVENT_ATTR(cid_write_retired, ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED),
0183     ARMV8_EVENT_ATTR(pc_write_retired, ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED),
0184     ARMV8_EVENT_ATTR(br_immed_retired, ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED),
0185     ARMV8_EVENT_ATTR(br_return_retired, ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED),
0186     ARMV8_EVENT_ATTR(unaligned_ldst_retired, ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED),
0187     ARMV8_EVENT_ATTR(br_mis_pred, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED),
0188     ARMV8_EVENT_ATTR(cpu_cycles, ARMV8_PMUV3_PERFCTR_CPU_CYCLES),
0189     ARMV8_EVENT_ATTR(br_pred, ARMV8_PMUV3_PERFCTR_BR_PRED),
0190     ARMV8_EVENT_ATTR(mem_access, ARMV8_PMUV3_PERFCTR_MEM_ACCESS),
0191     ARMV8_EVENT_ATTR(l1i_cache, ARMV8_PMUV3_PERFCTR_L1I_CACHE),
0192     ARMV8_EVENT_ATTR(l1d_cache_wb, ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB),
0193     ARMV8_EVENT_ATTR(l2d_cache, ARMV8_PMUV3_PERFCTR_L2D_CACHE),
0194     ARMV8_EVENT_ATTR(l2d_cache_refill, ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL),
0195     ARMV8_EVENT_ATTR(l2d_cache_wb, ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB),
0196     ARMV8_EVENT_ATTR(bus_access, ARMV8_PMUV3_PERFCTR_BUS_ACCESS),
0197     ARMV8_EVENT_ATTR(memory_error, ARMV8_PMUV3_PERFCTR_MEMORY_ERROR),
0198     ARMV8_EVENT_ATTR(inst_spec, ARMV8_PMUV3_PERFCTR_INST_SPEC),
0199     ARMV8_EVENT_ATTR(ttbr_write_retired, ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED),
0200     ARMV8_EVENT_ATTR(bus_cycles, ARMV8_PMUV3_PERFCTR_BUS_CYCLES),
0201     /* Don't expose the chain event in /sys, since it's useless in isolation */
0202     ARMV8_EVENT_ATTR(l1d_cache_allocate, ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE),
0203     ARMV8_EVENT_ATTR(l2d_cache_allocate, ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE),
0204     ARMV8_EVENT_ATTR(br_retired, ARMV8_PMUV3_PERFCTR_BR_RETIRED),
0205     ARMV8_EVENT_ATTR(br_mis_pred_retired, ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED),
0206     ARMV8_EVENT_ATTR(stall_frontend, ARMV8_PMUV3_PERFCTR_STALL_FRONTEND),
0207     ARMV8_EVENT_ATTR(stall_backend, ARMV8_PMUV3_PERFCTR_STALL_BACKEND),
0208     ARMV8_EVENT_ATTR(l1d_tlb, ARMV8_PMUV3_PERFCTR_L1D_TLB),
0209     ARMV8_EVENT_ATTR(l1i_tlb, ARMV8_PMUV3_PERFCTR_L1I_TLB),
0210     ARMV8_EVENT_ATTR(l2i_cache, ARMV8_PMUV3_PERFCTR_L2I_CACHE),
0211     ARMV8_EVENT_ATTR(l2i_cache_refill, ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL),
0212     ARMV8_EVENT_ATTR(l3d_cache_allocate, ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE),
0213     ARMV8_EVENT_ATTR(l3d_cache_refill, ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL),
0214     ARMV8_EVENT_ATTR(l3d_cache, ARMV8_PMUV3_PERFCTR_L3D_CACHE),
0215     ARMV8_EVENT_ATTR(l3d_cache_wb, ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB),
0216     ARMV8_EVENT_ATTR(l2d_tlb_refill, ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL),
0217     ARMV8_EVENT_ATTR(l2i_tlb_refill, ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL),
0218     ARMV8_EVENT_ATTR(l2d_tlb, ARMV8_PMUV3_PERFCTR_L2D_TLB),
0219     ARMV8_EVENT_ATTR(l2i_tlb, ARMV8_PMUV3_PERFCTR_L2I_TLB),
0220     ARMV8_EVENT_ATTR(remote_access, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS),
0221     ARMV8_EVENT_ATTR(ll_cache, ARMV8_PMUV3_PERFCTR_LL_CACHE),
0222     ARMV8_EVENT_ATTR(ll_cache_miss, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS),
0223     ARMV8_EVENT_ATTR(dtlb_walk, ARMV8_PMUV3_PERFCTR_DTLB_WALK),
0224     ARMV8_EVENT_ATTR(itlb_walk, ARMV8_PMUV3_PERFCTR_ITLB_WALK),
0225     ARMV8_EVENT_ATTR(ll_cache_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_RD),
0226     ARMV8_EVENT_ATTR(ll_cache_miss_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD),
0227     ARMV8_EVENT_ATTR(remote_access_rd, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS_RD),
0228     ARMV8_EVENT_ATTR(l1d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L1D_CACHE_LMISS_RD),
0229     ARMV8_EVENT_ATTR(op_retired, ARMV8_PMUV3_PERFCTR_OP_RETIRED),
0230     ARMV8_EVENT_ATTR(op_spec, ARMV8_PMUV3_PERFCTR_OP_SPEC),
0231     ARMV8_EVENT_ATTR(stall, ARMV8_PMUV3_PERFCTR_STALL),
0232     ARMV8_EVENT_ATTR(stall_slot_backend, ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND),
0233     ARMV8_EVENT_ATTR(stall_slot_frontend, ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND),
0234     ARMV8_EVENT_ATTR(stall_slot, ARMV8_PMUV3_PERFCTR_STALL_SLOT),
0235     ARMV8_EVENT_ATTR(sample_pop, ARMV8_SPE_PERFCTR_SAMPLE_POP),
0236     ARMV8_EVENT_ATTR(sample_feed, ARMV8_SPE_PERFCTR_SAMPLE_FEED),
0237     ARMV8_EVENT_ATTR(sample_filtrate, ARMV8_SPE_PERFCTR_SAMPLE_FILTRATE),
0238     ARMV8_EVENT_ATTR(sample_collision, ARMV8_SPE_PERFCTR_SAMPLE_COLLISION),
0239     ARMV8_EVENT_ATTR(cnt_cycles, ARMV8_AMU_PERFCTR_CNT_CYCLES),
0240     ARMV8_EVENT_ATTR(stall_backend_mem, ARMV8_AMU_PERFCTR_STALL_BACKEND_MEM),
0241     ARMV8_EVENT_ATTR(l1i_cache_lmiss, ARMV8_PMUV3_PERFCTR_L1I_CACHE_LMISS),
0242     ARMV8_EVENT_ATTR(l2d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L2D_CACHE_LMISS_RD),
0243     ARMV8_EVENT_ATTR(l2i_cache_lmiss, ARMV8_PMUV3_PERFCTR_L2I_CACHE_LMISS),
0244     ARMV8_EVENT_ATTR(l3d_cache_lmiss_rd, ARMV8_PMUV3_PERFCTR_L3D_CACHE_LMISS_RD),
0245     ARMV8_EVENT_ATTR(trb_wrap, ARMV8_PMUV3_PERFCTR_TRB_WRAP),
0246     ARMV8_EVENT_ATTR(trb_trig, ARMV8_PMUV3_PERFCTR_TRB_TRIG),
0247     ARMV8_EVENT_ATTR(trcextout0, ARMV8_PMUV3_PERFCTR_TRCEXTOUT0),
0248     ARMV8_EVENT_ATTR(trcextout1, ARMV8_PMUV3_PERFCTR_TRCEXTOUT1),
0249     ARMV8_EVENT_ATTR(trcextout2, ARMV8_PMUV3_PERFCTR_TRCEXTOUT2),
0250     ARMV8_EVENT_ATTR(trcextout3, ARMV8_PMUV3_PERFCTR_TRCEXTOUT3),
0251     ARMV8_EVENT_ATTR(cti_trigout4, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT4),
0252     ARMV8_EVENT_ATTR(cti_trigout5, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT5),
0253     ARMV8_EVENT_ATTR(cti_trigout6, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT6),
0254     ARMV8_EVENT_ATTR(cti_trigout7, ARMV8_PMUV3_PERFCTR_CTI_TRIGOUT7),
0255     ARMV8_EVENT_ATTR(ldst_align_lat, ARMV8_PMUV3_PERFCTR_LDST_ALIGN_LAT),
0256     ARMV8_EVENT_ATTR(ld_align_lat, ARMV8_PMUV3_PERFCTR_LD_ALIGN_LAT),
0257     ARMV8_EVENT_ATTR(st_align_lat, ARMV8_PMUV3_PERFCTR_ST_ALIGN_LAT),
0258     ARMV8_EVENT_ATTR(mem_access_checked, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED),
0259     ARMV8_EVENT_ATTR(mem_access_checked_rd, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED_RD),
0260     ARMV8_EVENT_ATTR(mem_access_checked_wr, ARMV8_MTE_PERFCTR_MEM_ACCESS_CHECKED_WR),
0261     NULL,
0262 };
0263 
0264 static umode_t
0265 armv8pmu_event_attr_is_visible(struct kobject *kobj,
0266                    struct attribute *attr, int unused)
0267 {
0268     struct device *dev = kobj_to_dev(kobj);
0269     struct pmu *pmu = dev_get_drvdata(dev);
0270     struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
0271     struct perf_pmu_events_attr *pmu_attr;
0272 
0273     pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
0274 
0275     if (pmu_attr->id < ARMV8_PMUV3_MAX_COMMON_EVENTS &&
0276         test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))
0277         return attr->mode;
0278 
0279     if (pmu_attr->id >= ARMV8_PMUV3_EXT_COMMON_EVENT_BASE) {
0280         u64 id = pmu_attr->id - ARMV8_PMUV3_EXT_COMMON_EVENT_BASE;
0281 
0282         if (id < ARMV8_PMUV3_MAX_COMMON_EVENTS &&
0283             test_bit(id, cpu_pmu->pmceid_ext_bitmap))
0284             return attr->mode;
0285     }
0286 
0287     return 0;
0288 }
0289 
0290 static const struct attribute_group armv8_pmuv3_events_attr_group = {
0291     .name = "events",
0292     .attrs = armv8_pmuv3_event_attrs,
0293     .is_visible = armv8pmu_event_attr_is_visible,
0294 };
0295 
0296 PMU_FORMAT_ATTR(event, "config:0-15");
0297 PMU_FORMAT_ATTR(long, "config1:0");
0298 PMU_FORMAT_ATTR(rdpmc, "config1:1");
0299 
0300 static int sysctl_perf_user_access __read_mostly;
0301 
0302 static inline bool armv8pmu_event_is_64bit(struct perf_event *event)
0303 {
0304     return event->attr.config1 & 0x1;
0305 }
0306 
0307 static inline bool armv8pmu_event_want_user_access(struct perf_event *event)
0308 {
0309     return event->attr.config1 & 0x2;
0310 }
0311 
0312 static struct attribute *armv8_pmuv3_format_attrs[] = {
0313     &format_attr_event.attr,
0314     &format_attr_long.attr,
0315     &format_attr_rdpmc.attr,
0316     NULL,
0317 };
0318 
0319 static const struct attribute_group armv8_pmuv3_format_attr_group = {
0320     .name = "format",
0321     .attrs = armv8_pmuv3_format_attrs,
0322 };
0323 
0324 static ssize_t slots_show(struct device *dev, struct device_attribute *attr,
0325               char *page)
0326 {
0327     struct pmu *pmu = dev_get_drvdata(dev);
0328     struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
0329     u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK;
0330 
0331     return sysfs_emit(page, "0x%08x\n", slots);
0332 }
0333 
0334 static DEVICE_ATTR_RO(slots);
0335 
0336 static ssize_t bus_slots_show(struct device *dev, struct device_attribute *attr,
0337                   char *page)
0338 {
0339     struct pmu *pmu = dev_get_drvdata(dev);
0340     struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
0341     u32 bus_slots = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_SLOTS_SHIFT)
0342             & ARMV8_PMU_BUS_SLOTS_MASK;
0343 
0344     return sysfs_emit(page, "0x%08x\n", bus_slots);
0345 }
0346 
0347 static DEVICE_ATTR_RO(bus_slots);
0348 
0349 static ssize_t bus_width_show(struct device *dev, struct device_attribute *attr,
0350                   char *page)
0351 {
0352     struct pmu *pmu = dev_get_drvdata(dev);
0353     struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
0354     u32 bus_width = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_WIDTH_SHIFT)
0355             & ARMV8_PMU_BUS_WIDTH_MASK;
0356     u32 val = 0;
0357 
0358     /* Encoded as Log2(number of bytes), plus one */
0359     if (bus_width > 2 && bus_width < 13)
0360         val = 1 << (bus_width - 1);
0361 
0362     return sysfs_emit(page, "0x%08x\n", val);
0363 }
0364 
0365 static DEVICE_ATTR_RO(bus_width);
0366 
0367 static struct attribute *armv8_pmuv3_caps_attrs[] = {
0368     &dev_attr_slots.attr,
0369     &dev_attr_bus_slots.attr,
0370     &dev_attr_bus_width.attr,
0371     NULL,
0372 };
0373 
0374 static const struct attribute_group armv8_pmuv3_caps_attr_group = {
0375     .name = "caps",
0376     .attrs = armv8_pmuv3_caps_attrs,
0377 };
0378 
0379 /*
0380  * Perf Events' indices
0381  */
0382 #define ARMV8_IDX_CYCLE_COUNTER 0
0383 #define ARMV8_IDX_COUNTER0  1
0384 #define ARMV8_IDX_CYCLE_COUNTER_USER    32
0385 
0386 /*
0387  * We unconditionally enable ARMv8.5-PMU long event counter support
0388  * (64-bit events) where supported. Indicate if this arm_pmu has long
0389  * event counter support.
0390  */
0391 static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu)
0392 {
0393     return (cpu_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_5);
0394 }
0395 
0396 static inline bool armv8pmu_event_has_user_read(struct perf_event *event)
0397 {
0398     return event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT;
0399 }
0400 
0401 /*
0402  * We must chain two programmable counters for 64 bit events,
0403  * except when we have allocated the 64bit cycle counter (for CPU
0404  * cycles event) or when user space counter access is enabled.
0405  */
0406 static inline bool armv8pmu_event_is_chained(struct perf_event *event)
0407 {
0408     int idx = event->hw.idx;
0409     struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
0410 
0411     return !armv8pmu_event_has_user_read(event) &&
0412            armv8pmu_event_is_64bit(event) &&
0413            !armv8pmu_has_long_event(cpu_pmu) &&
0414            (idx != ARMV8_IDX_CYCLE_COUNTER);
0415 }
0416 
0417 /*
0418  * ARMv8 low level PMU access
0419  */
0420 
0421 /*
0422  * Perf Event to low level counters mapping
0423  */
0424 #define ARMV8_IDX_TO_COUNTER(x) \
0425     (((x) - ARMV8_IDX_COUNTER0) & ARMV8_PMU_COUNTER_MASK)
0426 
0427 /*
0428  * This code is really good
0429  */
0430 
0431 #define PMEVN_CASE(n, case_macro) \
0432     case n: case_macro(n); break
0433 
0434 #define PMEVN_SWITCH(x, case_macro)             \
0435     do {                            \
0436         switch (x) {                    \
0437         PMEVN_CASE(0,  case_macro);         \
0438         PMEVN_CASE(1,  case_macro);         \
0439         PMEVN_CASE(2,  case_macro);         \
0440         PMEVN_CASE(3,  case_macro);         \
0441         PMEVN_CASE(4,  case_macro);         \
0442         PMEVN_CASE(5,  case_macro);         \
0443         PMEVN_CASE(6,  case_macro);         \
0444         PMEVN_CASE(7,  case_macro);         \
0445         PMEVN_CASE(8,  case_macro);         \
0446         PMEVN_CASE(9,  case_macro);         \
0447         PMEVN_CASE(10, case_macro);         \
0448         PMEVN_CASE(11, case_macro);         \
0449         PMEVN_CASE(12, case_macro);         \
0450         PMEVN_CASE(13, case_macro);         \
0451         PMEVN_CASE(14, case_macro);         \
0452         PMEVN_CASE(15, case_macro);         \
0453         PMEVN_CASE(16, case_macro);         \
0454         PMEVN_CASE(17, case_macro);         \
0455         PMEVN_CASE(18, case_macro);         \
0456         PMEVN_CASE(19, case_macro);         \
0457         PMEVN_CASE(20, case_macro);         \
0458         PMEVN_CASE(21, case_macro);         \
0459         PMEVN_CASE(22, case_macro);         \
0460         PMEVN_CASE(23, case_macro);         \
0461         PMEVN_CASE(24, case_macro);         \
0462         PMEVN_CASE(25, case_macro);         \
0463         PMEVN_CASE(26, case_macro);         \
0464         PMEVN_CASE(27, case_macro);         \
0465         PMEVN_CASE(28, case_macro);         \
0466         PMEVN_CASE(29, case_macro);         \
0467         PMEVN_CASE(30, case_macro);         \
0468         default: WARN(1, "Invalid PMEV* index\n");  \
0469         }                       \
0470     } while (0)
0471 
0472 #define RETURN_READ_PMEVCNTRN(n) \
0473     return read_sysreg(pmevcntr##n##_el0)
0474 static unsigned long read_pmevcntrn(int n)
0475 {
0476     PMEVN_SWITCH(n, RETURN_READ_PMEVCNTRN);
0477     return 0;
0478 }
0479 
0480 #define WRITE_PMEVCNTRN(n) \
0481     write_sysreg(val, pmevcntr##n##_el0)
0482 static void write_pmevcntrn(int n, unsigned long val)
0483 {
0484     PMEVN_SWITCH(n, WRITE_PMEVCNTRN);
0485 }
0486 
0487 #define WRITE_PMEVTYPERN(n) \
0488     write_sysreg(val, pmevtyper##n##_el0)
0489 static void write_pmevtypern(int n, unsigned long val)
0490 {
0491     PMEVN_SWITCH(n, WRITE_PMEVTYPERN);
0492 }
0493 
0494 static inline u32 armv8pmu_pmcr_read(void)
0495 {
0496     return read_sysreg(pmcr_el0);
0497 }
0498 
0499 static inline void armv8pmu_pmcr_write(u32 val)
0500 {
0501     val &= ARMV8_PMU_PMCR_MASK;
0502     isb();
0503     write_sysreg(val, pmcr_el0);
0504 }
0505 
0506 static inline int armv8pmu_has_overflowed(u32 pmovsr)
0507 {
0508     return pmovsr & ARMV8_PMU_OVERFLOWED_MASK;
0509 }
0510 
0511 static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx)
0512 {
0513     return pmnc & BIT(ARMV8_IDX_TO_COUNTER(idx));
0514 }
0515 
0516 static inline u64 armv8pmu_read_evcntr(int idx)
0517 {
0518     u32 counter = ARMV8_IDX_TO_COUNTER(idx);
0519 
0520     return read_pmevcntrn(counter);
0521 }
0522 
0523 static inline u64 armv8pmu_read_hw_counter(struct perf_event *event)
0524 {
0525     int idx = event->hw.idx;
0526     u64 val = armv8pmu_read_evcntr(idx);
0527 
0528     if (armv8pmu_event_is_chained(event))
0529         val = (val << 32) | armv8pmu_read_evcntr(idx - 1);
0530     return val;
0531 }
0532 
0533 /*
0534  * The cycle counter is always a 64-bit counter. When ARMV8_PMU_PMCR_LP
0535  * is set the event counters also become 64-bit counters. Unless the
0536  * user has requested a long counter (attr.config1) then we want to
0537  * interrupt upon 32-bit overflow - we achieve this by applying a bias.
0538  */
0539 static bool armv8pmu_event_needs_bias(struct perf_event *event)
0540 {
0541     struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
0542     struct hw_perf_event *hwc = &event->hw;
0543     int idx = hwc->idx;
0544 
0545     if (armv8pmu_event_is_64bit(event))
0546         return false;
0547 
0548     if (armv8pmu_has_long_event(cpu_pmu) ||
0549         idx == ARMV8_IDX_CYCLE_COUNTER)
0550         return true;
0551 
0552     return false;
0553 }
0554 
0555 static u64 armv8pmu_bias_long_counter(struct perf_event *event, u64 value)
0556 {
0557     if (armv8pmu_event_needs_bias(event))
0558         value |= GENMASK(63, 32);
0559 
0560     return value;
0561 }
0562 
0563 static u64 armv8pmu_unbias_long_counter(struct perf_event *event, u64 value)
0564 {
0565     if (armv8pmu_event_needs_bias(event))
0566         value &= ~GENMASK(63, 32);
0567 
0568     return value;
0569 }
0570 
0571 static u64 armv8pmu_read_counter(struct perf_event *event)
0572 {
0573     struct hw_perf_event *hwc = &event->hw;
0574     int idx = hwc->idx;
0575     u64 value;
0576 
0577     if (idx == ARMV8_IDX_CYCLE_COUNTER)
0578         value = read_sysreg(pmccntr_el0);
0579     else
0580         value = armv8pmu_read_hw_counter(event);
0581 
0582     return  armv8pmu_unbias_long_counter(event, value);
0583 }
0584 
0585 static inline void armv8pmu_write_evcntr(int idx, u64 value)
0586 {
0587     u32 counter = ARMV8_IDX_TO_COUNTER(idx);
0588 
0589     write_pmevcntrn(counter, value);
0590 }
0591 
0592 static inline void armv8pmu_write_hw_counter(struct perf_event *event,
0593                          u64 value)
0594 {
0595     int idx = event->hw.idx;
0596 
0597     if (armv8pmu_event_is_chained(event)) {
0598         armv8pmu_write_evcntr(idx, upper_32_bits(value));
0599         armv8pmu_write_evcntr(idx - 1, lower_32_bits(value));
0600     } else {
0601         armv8pmu_write_evcntr(idx, value);
0602     }
0603 }
0604 
0605 static void armv8pmu_write_counter(struct perf_event *event, u64 value)
0606 {
0607     struct hw_perf_event *hwc = &event->hw;
0608     int idx = hwc->idx;
0609 
0610     value = armv8pmu_bias_long_counter(event, value);
0611 
0612     if (idx == ARMV8_IDX_CYCLE_COUNTER)
0613         write_sysreg(value, pmccntr_el0);
0614     else
0615         armv8pmu_write_hw_counter(event, value);
0616 }
0617 
0618 static inline void armv8pmu_write_evtype(int idx, u32 val)
0619 {
0620     u32 counter = ARMV8_IDX_TO_COUNTER(idx);
0621 
0622     val &= ARMV8_PMU_EVTYPE_MASK;
0623     write_pmevtypern(counter, val);
0624 }
0625 
0626 static inline void armv8pmu_write_event_type(struct perf_event *event)
0627 {
0628     struct hw_perf_event *hwc = &event->hw;
0629     int idx = hwc->idx;
0630 
0631     /*
0632      * For chained events, the low counter is programmed to count
0633      * the event of interest and the high counter is programmed
0634      * with CHAIN event code with filters set to count at all ELs.
0635      */
0636     if (armv8pmu_event_is_chained(event)) {
0637         u32 chain_evt = ARMV8_PMUV3_PERFCTR_CHAIN |
0638                 ARMV8_PMU_INCLUDE_EL2;
0639 
0640         armv8pmu_write_evtype(idx - 1, hwc->config_base);
0641         armv8pmu_write_evtype(idx, chain_evt);
0642     } else {
0643         if (idx == ARMV8_IDX_CYCLE_COUNTER)
0644             write_sysreg(hwc->config_base, pmccfiltr_el0);
0645         else
0646             armv8pmu_write_evtype(idx, hwc->config_base);
0647     }
0648 }
0649 
0650 static u32 armv8pmu_event_cnten_mask(struct perf_event *event)
0651 {
0652     int counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
0653     u32 mask = BIT(counter);
0654 
0655     if (armv8pmu_event_is_chained(event))
0656         mask |= BIT(counter - 1);
0657     return mask;
0658 }
0659 
0660 static inline void armv8pmu_enable_counter(u32 mask)
0661 {
0662     /*
0663      * Make sure event configuration register writes are visible before we
0664      * enable the counter.
0665      * */
0666     isb();
0667     write_sysreg(mask, pmcntenset_el0);
0668 }
0669 
0670 static inline void armv8pmu_enable_event_counter(struct perf_event *event)
0671 {
0672     struct perf_event_attr *attr = &event->attr;
0673     u32 mask = armv8pmu_event_cnten_mask(event);
0674 
0675     kvm_set_pmu_events(mask, attr);
0676 
0677     /* We rely on the hypervisor switch code to enable guest counters */
0678     if (!kvm_pmu_counter_deferred(attr))
0679         armv8pmu_enable_counter(mask);
0680 }
0681 
0682 static inline void armv8pmu_disable_counter(u32 mask)
0683 {
0684     write_sysreg(mask, pmcntenclr_el0);
0685     /*
0686      * Make sure the effects of disabling the counter are visible before we
0687      * start configuring the event.
0688      */
0689     isb();
0690 }
0691 
0692 static inline void armv8pmu_disable_event_counter(struct perf_event *event)
0693 {
0694     struct perf_event_attr *attr = &event->attr;
0695     u32 mask = armv8pmu_event_cnten_mask(event);
0696 
0697     kvm_clr_pmu_events(mask);
0698 
0699     /* We rely on the hypervisor switch code to disable guest counters */
0700     if (!kvm_pmu_counter_deferred(attr))
0701         armv8pmu_disable_counter(mask);
0702 }
0703 
0704 static inline void armv8pmu_enable_intens(u32 mask)
0705 {
0706     write_sysreg(mask, pmintenset_el1);
0707 }
0708 
0709 static inline void armv8pmu_enable_event_irq(struct perf_event *event)
0710 {
0711     u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
0712     armv8pmu_enable_intens(BIT(counter));
0713 }
0714 
0715 static inline void armv8pmu_disable_intens(u32 mask)
0716 {
0717     write_sysreg(mask, pmintenclr_el1);
0718     isb();
0719     /* Clear the overflow flag in case an interrupt is pending. */
0720     write_sysreg(mask, pmovsclr_el0);
0721     isb();
0722 }
0723 
0724 static inline void armv8pmu_disable_event_irq(struct perf_event *event)
0725 {
0726     u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
0727     armv8pmu_disable_intens(BIT(counter));
0728 }
0729 
0730 static inline u32 armv8pmu_getreset_flags(void)
0731 {
0732     u32 value;
0733 
0734     /* Read */
0735     value = read_sysreg(pmovsclr_el0);
0736 
0737     /* Write to clear flags */
0738     value &= ARMV8_PMU_OVSR_MASK;
0739     write_sysreg(value, pmovsclr_el0);
0740 
0741     return value;
0742 }
0743 
0744 static void armv8pmu_disable_user_access(void)
0745 {
0746     write_sysreg(0, pmuserenr_el0);
0747 }
0748 
0749 static void armv8pmu_enable_user_access(struct arm_pmu *cpu_pmu)
0750 {
0751     int i;
0752     struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
0753 
0754     /* Clear any unused counters to avoid leaking their contents */
0755     for_each_clear_bit(i, cpuc->used_mask, cpu_pmu->num_events) {
0756         if (i == ARMV8_IDX_CYCLE_COUNTER)
0757             write_sysreg(0, pmccntr_el0);
0758         else
0759             armv8pmu_write_evcntr(i, 0);
0760     }
0761 
0762     write_sysreg(0, pmuserenr_el0);
0763     write_sysreg(ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR, pmuserenr_el0);
0764 }
0765 
0766 static void armv8pmu_enable_event(struct perf_event *event)
0767 {
0768     /*
0769      * Enable counter and interrupt, and set the counter to count
0770      * the event that we're interested in.
0771      */
0772 
0773     /*
0774      * Disable counter
0775      */
0776     armv8pmu_disable_event_counter(event);
0777 
0778     /*
0779      * Set event.
0780      */
0781     armv8pmu_write_event_type(event);
0782 
0783     /*
0784      * Enable interrupt for this counter
0785      */
0786     armv8pmu_enable_event_irq(event);
0787 
0788     /*
0789      * Enable counter
0790      */
0791     armv8pmu_enable_event_counter(event);
0792 }
0793 
0794 static void armv8pmu_disable_event(struct perf_event *event)
0795 {
0796     /*
0797      * Disable counter
0798      */
0799     armv8pmu_disable_event_counter(event);
0800 
0801     /*
0802      * Disable interrupt for this counter
0803      */
0804     armv8pmu_disable_event_irq(event);
0805 }
0806 
0807 static void armv8pmu_start(struct arm_pmu *cpu_pmu)
0808 {
0809     struct perf_event_context *task_ctx =
0810         this_cpu_ptr(cpu_pmu->pmu.pmu_cpu_context)->task_ctx;
0811 
0812     if (sysctl_perf_user_access && task_ctx && task_ctx->nr_user)
0813         armv8pmu_enable_user_access(cpu_pmu);
0814     else
0815         armv8pmu_disable_user_access();
0816 
0817     /* Enable all counters */
0818     armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMU_PMCR_E);
0819 }
0820 
0821 static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
0822 {
0823     /* Disable all counters */
0824     armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
0825 }
0826 
0827 static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)
0828 {
0829     u32 pmovsr;
0830     struct perf_sample_data data;
0831     struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
0832     struct pt_regs *regs;
0833     int idx;
0834 
0835     /*
0836      * Get and reset the IRQ flags
0837      */
0838     pmovsr = armv8pmu_getreset_flags();
0839 
0840     /*
0841      * Did an overflow occur?
0842      */
0843     if (!armv8pmu_has_overflowed(pmovsr))
0844         return IRQ_NONE;
0845 
0846     /*
0847      * Handle the counter(s) overflow(s)
0848      */
0849     regs = get_irq_regs();
0850 
0851     /*
0852      * Stop the PMU while processing the counter overflows
0853      * to prevent skews in group events.
0854      */
0855     armv8pmu_stop(cpu_pmu);
0856     for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
0857         struct perf_event *event = cpuc->events[idx];
0858         struct hw_perf_event *hwc;
0859 
0860         /* Ignore if we don't have an event. */
0861         if (!event)
0862             continue;
0863 
0864         /*
0865          * We have a single interrupt for all counters. Check that
0866          * each counter has overflowed before we process it.
0867          */
0868         if (!armv8pmu_counter_has_overflowed(pmovsr, idx))
0869             continue;
0870 
0871         hwc = &event->hw;
0872         armpmu_event_update(event);
0873         perf_sample_data_init(&data, 0, hwc->last_period);
0874         if (!armpmu_event_set_period(event))
0875             continue;
0876 
0877         /*
0878          * Perf event overflow will queue the processing of the event as
0879          * an irq_work which will be taken care of in the handling of
0880          * IPI_IRQ_WORK.
0881          */
0882         if (perf_event_overflow(event, &data, regs))
0883             cpu_pmu->disable(event);
0884     }
0885     armv8pmu_start(cpu_pmu);
0886 
0887     return IRQ_HANDLED;
0888 }
0889 
0890 static int armv8pmu_get_single_idx(struct pmu_hw_events *cpuc,
0891                     struct arm_pmu *cpu_pmu)
0892 {
0893     int idx;
0894 
0895     for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; idx++) {
0896         if (!test_and_set_bit(idx, cpuc->used_mask))
0897             return idx;
0898     }
0899     return -EAGAIN;
0900 }
0901 
0902 static int armv8pmu_get_chain_idx(struct pmu_hw_events *cpuc,
0903                    struct arm_pmu *cpu_pmu)
0904 {
0905     int idx;
0906 
0907     /*
0908      * Chaining requires two consecutive event counters, where
0909      * the lower idx must be even.
0910      */
0911     for (idx = ARMV8_IDX_COUNTER0 + 1; idx < cpu_pmu->num_events; idx += 2) {
0912         if (!test_and_set_bit(idx, cpuc->used_mask)) {
0913             /* Check if the preceding even counter is available */
0914             if (!test_and_set_bit(idx - 1, cpuc->used_mask))
0915                 return idx;
0916             /* Release the Odd counter */
0917             clear_bit(idx, cpuc->used_mask);
0918         }
0919     }
0920     return -EAGAIN;
0921 }
0922 
0923 static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
0924                   struct perf_event *event)
0925 {
0926     struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
0927     struct hw_perf_event *hwc = &event->hw;
0928     unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;
0929 
0930     /* Always prefer to place a cycle counter into the cycle counter. */
0931     if (evtype == ARMV8_PMUV3_PERFCTR_CPU_CYCLES) {
0932         if (!test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))
0933             return ARMV8_IDX_CYCLE_COUNTER;
0934         else if (armv8pmu_event_is_64bit(event) &&
0935                armv8pmu_event_want_user_access(event) &&
0936                !armv8pmu_has_long_event(cpu_pmu))
0937                 return -EAGAIN;
0938     }
0939 
0940     /*
0941      * Otherwise use events counters
0942      */
0943     if (armv8pmu_event_is_chained(event))
0944         return  armv8pmu_get_chain_idx(cpuc, cpu_pmu);
0945     else
0946         return armv8pmu_get_single_idx(cpuc, cpu_pmu);
0947 }
0948 
0949 static void armv8pmu_clear_event_idx(struct pmu_hw_events *cpuc,
0950                      struct perf_event *event)
0951 {
0952     int idx = event->hw.idx;
0953 
0954     clear_bit(idx, cpuc->used_mask);
0955     if (armv8pmu_event_is_chained(event))
0956         clear_bit(idx - 1, cpuc->used_mask);
0957 }
0958 
0959 static int armv8pmu_user_event_idx(struct perf_event *event)
0960 {
0961     if (!sysctl_perf_user_access || !armv8pmu_event_has_user_read(event))
0962         return 0;
0963 
0964     /*
0965      * We remap the cycle counter index to 32 to
0966      * match the offset applied to the rest of
0967      * the counter indices.
0968      */
0969     if (event->hw.idx == ARMV8_IDX_CYCLE_COUNTER)
0970         return ARMV8_IDX_CYCLE_COUNTER_USER;
0971 
0972     return event->hw.idx;
0973 }
0974 
0975 /*
0976  * Add an event filter to a given event.
0977  */
0978 static int armv8pmu_set_event_filter(struct hw_perf_event *event,
0979                      struct perf_event_attr *attr)
0980 {
0981     unsigned long config_base = 0;
0982 
0983     if (attr->exclude_idle)
0984         return -EPERM;
0985 
0986     /*
0987      * If we're running in hyp mode, then we *are* the hypervisor.
0988      * Therefore we ignore exclude_hv in this configuration, since
0989      * there's no hypervisor to sample anyway. This is consistent
0990      * with other architectures (x86 and Power).
0991      */
0992     if (is_kernel_in_hyp_mode()) {
0993         if (!attr->exclude_kernel && !attr->exclude_host)
0994             config_base |= ARMV8_PMU_INCLUDE_EL2;
0995         if (attr->exclude_guest)
0996             config_base |= ARMV8_PMU_EXCLUDE_EL1;
0997         if (attr->exclude_host)
0998             config_base |= ARMV8_PMU_EXCLUDE_EL0;
0999     } else {
1000         if (!attr->exclude_hv && !attr->exclude_host)
1001             config_base |= ARMV8_PMU_INCLUDE_EL2;
1002     }
1003 
1004     /*
1005      * Filter out !VHE kernels and guest kernels
1006      */
1007     if (attr->exclude_kernel)
1008         config_base |= ARMV8_PMU_EXCLUDE_EL1;
1009 
1010     if (attr->exclude_user)
1011         config_base |= ARMV8_PMU_EXCLUDE_EL0;
1012 
1013     /*
1014      * Install the filter into config_base as this is used to
1015      * construct the event type.
1016      */
1017     event->config_base = config_base;
1018 
1019     return 0;
1020 }
1021 
1022 static int armv8pmu_filter_match(struct perf_event *event)
1023 {
1024     unsigned long evtype = event->hw.config_base & ARMV8_PMU_EVTYPE_EVENT;
1025     return evtype != ARMV8_PMUV3_PERFCTR_CHAIN;
1026 }
1027 
1028 static void armv8pmu_reset(void *info)
1029 {
1030     struct arm_pmu *cpu_pmu = (struct arm_pmu *)info;
1031     u32 pmcr;
1032 
1033     /* The counter and interrupt enable registers are unknown at reset. */
1034     armv8pmu_disable_counter(U32_MAX);
1035     armv8pmu_disable_intens(U32_MAX);
1036 
1037     /* Clear the counters we flip at guest entry/exit */
1038     kvm_clr_pmu_events(U32_MAX);
1039 
1040     /*
1041      * Initialize & Reset PMNC. Request overflow interrupt for
1042      * 64 bit cycle counter but cheat in armv8pmu_write_counter().
1043      */
1044     pmcr = ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C | ARMV8_PMU_PMCR_LC;
1045 
1046     /* Enable long event counter support where available */
1047     if (armv8pmu_has_long_event(cpu_pmu))
1048         pmcr |= ARMV8_PMU_PMCR_LP;
1049 
1050     armv8pmu_pmcr_write(pmcr);
1051 }
1052 
1053 static int __armv8_pmuv3_map_event(struct perf_event *event,
1054                    const unsigned (*extra_event_map)
1055                           [PERF_COUNT_HW_MAX],
1056                    const unsigned (*extra_cache_map)
1057                           [PERF_COUNT_HW_CACHE_MAX]
1058                           [PERF_COUNT_HW_CACHE_OP_MAX]
1059                           [PERF_COUNT_HW_CACHE_RESULT_MAX])
1060 {
1061     int hw_event_id;
1062     struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
1063 
1064     hw_event_id = armpmu_map_event(event, &armv8_pmuv3_perf_map,
1065                        &armv8_pmuv3_perf_cache_map,
1066                        ARMV8_PMU_EVTYPE_EVENT);
1067 
1068     if (armv8pmu_event_is_64bit(event))
1069         event->hw.flags |= ARMPMU_EVT_64BIT;
1070 
1071     /*
1072      * User events must be allocated into a single counter, and so
1073      * must not be chained.
1074      *
1075      * Most 64-bit events require long counter support, but 64-bit
1076      * CPU_CYCLES events can be placed into the dedicated cycle
1077      * counter when this is free.
1078      */
1079     if (armv8pmu_event_want_user_access(event)) {
1080         if (!(event->attach_state & PERF_ATTACH_TASK))
1081             return -EINVAL;
1082         if (armv8pmu_event_is_64bit(event) &&
1083             (hw_event_id != ARMV8_PMUV3_PERFCTR_CPU_CYCLES) &&
1084             !armv8pmu_has_long_event(armpmu))
1085             return -EOPNOTSUPP;
1086 
1087         event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT;
1088     }
1089 
1090     /* Only expose micro/arch events supported by this PMU */
1091     if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
1092         && test_bit(hw_event_id, armpmu->pmceid_bitmap)) {
1093         return hw_event_id;
1094     }
1095 
1096     return armpmu_map_event(event, extra_event_map, extra_cache_map,
1097                 ARMV8_PMU_EVTYPE_EVENT);
1098 }
1099 
1100 static int armv8_pmuv3_map_event(struct perf_event *event)
1101 {
1102     return __armv8_pmuv3_map_event(event, NULL, NULL);
1103 }
1104 
1105 static int armv8_a53_map_event(struct perf_event *event)
1106 {
1107     return __armv8_pmuv3_map_event(event, NULL, &armv8_a53_perf_cache_map);
1108 }
1109 
1110 static int armv8_a57_map_event(struct perf_event *event)
1111 {
1112     return __armv8_pmuv3_map_event(event, NULL, &armv8_a57_perf_cache_map);
1113 }
1114 
1115 static int armv8_a73_map_event(struct perf_event *event)
1116 {
1117     return __armv8_pmuv3_map_event(event, NULL, &armv8_a73_perf_cache_map);
1118 }
1119 
1120 static int armv8_thunder_map_event(struct perf_event *event)
1121 {
1122     return __armv8_pmuv3_map_event(event, NULL,
1123                        &armv8_thunder_perf_cache_map);
1124 }
1125 
1126 static int armv8_vulcan_map_event(struct perf_event *event)
1127 {
1128     return __armv8_pmuv3_map_event(event, NULL,
1129                        &armv8_vulcan_perf_cache_map);
1130 }
1131 
1132 struct armv8pmu_probe_info {
1133     struct arm_pmu *pmu;
1134     bool present;
1135 };
1136 
1137 static void __armv8pmu_probe_pmu(void *info)
1138 {
1139     struct armv8pmu_probe_info *probe = info;
1140     struct arm_pmu *cpu_pmu = probe->pmu;
1141     u64 dfr0;
1142     u64 pmceid_raw[2];
1143     u32 pmceid[2];
1144     int pmuver;
1145 
1146     dfr0 = read_sysreg(id_aa64dfr0_el1);
1147     pmuver = cpuid_feature_extract_unsigned_field(dfr0,
1148             ID_AA64DFR0_PMUVER_SHIFT);
1149     if (pmuver == ID_AA64DFR0_PMUVER_IMP_DEF || pmuver == 0)
1150         return;
1151 
1152     cpu_pmu->pmuver = pmuver;
1153     probe->present = true;
1154 
1155     /* Read the nb of CNTx counters supported from PMNC */
1156     cpu_pmu->num_events = (armv8pmu_pmcr_read() >> ARMV8_PMU_PMCR_N_SHIFT)
1157         & ARMV8_PMU_PMCR_N_MASK;
1158 
1159     /* Add the CPU cycles counter */
1160     cpu_pmu->num_events += 1;
1161 
1162     pmceid[0] = pmceid_raw[0] = read_sysreg(pmceid0_el0);
1163     pmceid[1] = pmceid_raw[1] = read_sysreg(pmceid1_el0);
1164 
1165     bitmap_from_arr32(cpu_pmu->pmceid_bitmap,
1166                  pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
1167 
1168     pmceid[0] = pmceid_raw[0] >> 32;
1169     pmceid[1] = pmceid_raw[1] >> 32;
1170 
1171     bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap,
1172                  pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
1173 
1174     /* store PMMIR_EL1 register for sysfs */
1175     if (pmuver >= ID_AA64DFR0_PMUVER_8_4 && (pmceid_raw[1] & BIT(31)))
1176         cpu_pmu->reg_pmmir = read_cpuid(PMMIR_EL1);
1177     else
1178         cpu_pmu->reg_pmmir = 0;
1179 }
1180 
1181 static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
1182 {
1183     struct armv8pmu_probe_info probe = {
1184         .pmu = cpu_pmu,
1185         .present = false,
1186     };
1187     int ret;
1188 
1189     ret = smp_call_function_any(&cpu_pmu->supported_cpus,
1190                     __armv8pmu_probe_pmu,
1191                     &probe, 1);
1192     if (ret)
1193         return ret;
1194 
1195     return probe.present ? 0 : -ENODEV;
1196 }
1197 
1198 static void armv8pmu_disable_user_access_ipi(void *unused)
1199 {
1200     armv8pmu_disable_user_access();
1201 }
1202 
1203 static int armv8pmu_proc_user_access_handler(struct ctl_table *table, int write,
1204         void *buffer, size_t *lenp, loff_t *ppos)
1205 {
1206     int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
1207     if (ret || !write || sysctl_perf_user_access)
1208         return ret;
1209 
1210     on_each_cpu(armv8pmu_disable_user_access_ipi, NULL, 1);
1211     return 0;
1212 }
1213 
1214 static struct ctl_table armv8_pmu_sysctl_table[] = {
1215     {
1216         .procname       = "perf_user_access",
1217         .data       = &sysctl_perf_user_access,
1218         .maxlen     = sizeof(unsigned int),
1219         .mode           = 0644,
1220         .proc_handler   = armv8pmu_proc_user_access_handler,
1221         .extra1     = SYSCTL_ZERO,
1222         .extra2     = SYSCTL_ONE,
1223     },
1224     { }
1225 };
1226 
1227 static void armv8_pmu_register_sysctl_table(void)
1228 {
1229     static u32 tbl_registered = 0;
1230 
1231     if (!cmpxchg_relaxed(&tbl_registered, 0, 1))
1232         register_sysctl("kernel", armv8_pmu_sysctl_table);
1233 }
1234 
1235 static int armv8_pmu_init(struct arm_pmu *cpu_pmu, char *name,
1236               int (*map_event)(struct perf_event *event),
1237               const struct attribute_group *events,
1238               const struct attribute_group *format,
1239               const struct attribute_group *caps)
1240 {
1241     int ret = armv8pmu_probe_pmu(cpu_pmu);
1242     if (ret)
1243         return ret;
1244 
1245     cpu_pmu->handle_irq     = armv8pmu_handle_irq;
1246     cpu_pmu->enable         = armv8pmu_enable_event;
1247     cpu_pmu->disable        = armv8pmu_disable_event;
1248     cpu_pmu->read_counter       = armv8pmu_read_counter;
1249     cpu_pmu->write_counter      = armv8pmu_write_counter;
1250     cpu_pmu->get_event_idx      = armv8pmu_get_event_idx;
1251     cpu_pmu->clear_event_idx    = armv8pmu_clear_event_idx;
1252     cpu_pmu->start          = armv8pmu_start;
1253     cpu_pmu->stop           = armv8pmu_stop;
1254     cpu_pmu->reset          = armv8pmu_reset;
1255     cpu_pmu->set_event_filter   = armv8pmu_set_event_filter;
1256     cpu_pmu->filter_match       = armv8pmu_filter_match;
1257 
1258     cpu_pmu->pmu.event_idx      = armv8pmu_user_event_idx;
1259 
1260     cpu_pmu->name           = name;
1261     cpu_pmu->map_event      = map_event;
1262     cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = events ?
1263             events : &armv8_pmuv3_events_attr_group;
1264     cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = format ?
1265             format : &armv8_pmuv3_format_attr_group;
1266     cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_CAPS] = caps ?
1267             caps : &armv8_pmuv3_caps_attr_group;
1268 
1269     armv8_pmu_register_sysctl_table();
1270     return 0;
1271 }
1272 
1273 static int armv8_pmu_init_nogroups(struct arm_pmu *cpu_pmu, char *name,
1274                    int (*map_event)(struct perf_event *event))
1275 {
1276     return armv8_pmu_init(cpu_pmu, name, map_event, NULL, NULL, NULL);
1277 }
1278 
1279 #define PMUV3_INIT_SIMPLE(name)                     \
1280 static int name##_pmu_init(struct arm_pmu *cpu_pmu)         \
1281 {                                   \
1282     return armv8_pmu_init_nogroups(cpu_pmu, #name, armv8_pmuv3_map_event);\
1283 }
1284 
1285 PMUV3_INIT_SIMPLE(armv8_pmuv3)
1286 
1287 PMUV3_INIT_SIMPLE(armv8_cortex_a34)
1288 PMUV3_INIT_SIMPLE(armv8_cortex_a55)
1289 PMUV3_INIT_SIMPLE(armv8_cortex_a65)
1290 PMUV3_INIT_SIMPLE(armv8_cortex_a75)
1291 PMUV3_INIT_SIMPLE(armv8_cortex_a76)
1292 PMUV3_INIT_SIMPLE(armv8_cortex_a77)
1293 PMUV3_INIT_SIMPLE(armv8_cortex_a78)
1294 PMUV3_INIT_SIMPLE(armv9_cortex_a510)
1295 PMUV3_INIT_SIMPLE(armv9_cortex_a710)
1296 PMUV3_INIT_SIMPLE(armv8_cortex_x1)
1297 PMUV3_INIT_SIMPLE(armv9_cortex_x2)
1298 PMUV3_INIT_SIMPLE(armv8_neoverse_e1)
1299 PMUV3_INIT_SIMPLE(armv8_neoverse_n1)
1300 PMUV3_INIT_SIMPLE(armv9_neoverse_n2)
1301 PMUV3_INIT_SIMPLE(armv8_neoverse_v1)
1302 
1303 PMUV3_INIT_SIMPLE(armv8_nvidia_carmel)
1304 PMUV3_INIT_SIMPLE(armv8_nvidia_denver)
1305 
1306 static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)
1307 {
1308     return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a35",
1309                        armv8_a53_map_event);
1310 }
1311 
1312 static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
1313 {
1314     return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a53",
1315                        armv8_a53_map_event);
1316 }
1317 
1318 static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)
1319 {
1320     return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a57",
1321                        armv8_a57_map_event);
1322 }
1323 
1324 static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)
1325 {
1326     return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a72",
1327                        armv8_a57_map_event);
1328 }
1329 
1330 static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)
1331 {
1332     return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cortex_a73",
1333                        armv8_a73_map_event);
1334 }
1335 
1336 static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
1337 {
1338     return armv8_pmu_init_nogroups(cpu_pmu, "armv8_cavium_thunder",
1339                        armv8_thunder_map_event);
1340 }
1341 
1342 static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)
1343 {
1344     return armv8_pmu_init_nogroups(cpu_pmu, "armv8_brcm_vulcan",
1345                        armv8_vulcan_map_event);
1346 }
1347 
1348 static const struct of_device_id armv8_pmu_of_device_ids[] = {
1349     {.compatible = "arm,armv8-pmuv3",   .data = armv8_pmuv3_pmu_init},
1350     {.compatible = "arm,cortex-a34-pmu",    .data = armv8_cortex_a34_pmu_init},
1351     {.compatible = "arm,cortex-a35-pmu",    .data = armv8_a35_pmu_init},
1352     {.compatible = "arm,cortex-a53-pmu",    .data = armv8_a53_pmu_init},
1353     {.compatible = "arm,cortex-a55-pmu",    .data = armv8_cortex_a55_pmu_init},
1354     {.compatible = "arm,cortex-a57-pmu",    .data = armv8_a57_pmu_init},
1355     {.compatible = "arm,cortex-a65-pmu",    .data = armv8_cortex_a65_pmu_init},
1356     {.compatible = "arm,cortex-a72-pmu",    .data = armv8_a72_pmu_init},
1357     {.compatible = "arm,cortex-a73-pmu",    .data = armv8_a73_pmu_init},
1358     {.compatible = "arm,cortex-a75-pmu",    .data = armv8_cortex_a75_pmu_init},
1359     {.compatible = "arm,cortex-a76-pmu",    .data = armv8_cortex_a76_pmu_init},
1360     {.compatible = "arm,cortex-a77-pmu",    .data = armv8_cortex_a77_pmu_init},
1361     {.compatible = "arm,cortex-a78-pmu",    .data = armv8_cortex_a78_pmu_init},
1362     {.compatible = "arm,cortex-a510-pmu",   .data = armv9_cortex_a510_pmu_init},
1363     {.compatible = "arm,cortex-a710-pmu",   .data = armv9_cortex_a710_pmu_init},
1364     {.compatible = "arm,cortex-x1-pmu", .data = armv8_cortex_x1_pmu_init},
1365     {.compatible = "arm,cortex-x2-pmu", .data = armv9_cortex_x2_pmu_init},
1366     {.compatible = "arm,neoverse-e1-pmu",   .data = armv8_neoverse_e1_pmu_init},
1367     {.compatible = "arm,neoverse-n1-pmu",   .data = armv8_neoverse_n1_pmu_init},
1368     {.compatible = "arm,neoverse-n2-pmu",   .data = armv9_neoverse_n2_pmu_init},
1369     {.compatible = "arm,neoverse-v1-pmu",   .data = armv8_neoverse_v1_pmu_init},
1370     {.compatible = "cavium,thunder-pmu",    .data = armv8_thunder_pmu_init},
1371     {.compatible = "brcm,vulcan-pmu",   .data = armv8_vulcan_pmu_init},
1372     {.compatible = "nvidia,carmel-pmu", .data = armv8_nvidia_carmel_pmu_init},
1373     {.compatible = "nvidia,denver-pmu", .data = armv8_nvidia_denver_pmu_init},
1374     {},
1375 };
1376 
1377 static int armv8_pmu_device_probe(struct platform_device *pdev)
1378 {
1379     return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids, NULL);
1380 }
1381 
1382 static struct platform_driver armv8_pmu_driver = {
1383     .driver     = {
1384         .name   = ARMV8_PMU_PDEV_NAME,
1385         .of_match_table = armv8_pmu_of_device_ids,
1386         .suppress_bind_attrs = true,
1387     },
1388     .probe      = armv8_pmu_device_probe,
1389 };
1390 
1391 static int __init armv8_pmu_driver_init(void)
1392 {
1393     if (acpi_disabled)
1394         return platform_driver_register(&armv8_pmu_driver);
1395     else
1396         return arm_pmu_acpi_probe(armv8_pmuv3_pmu_init);
1397 }
1398 device_initcall(armv8_pmu_driver_init)
1399 
1400 void arch_perf_update_userpage(struct perf_event *event,
1401                    struct perf_event_mmap_page *userpg, u64 now)
1402 {
1403     struct clock_read_data *rd;
1404     unsigned int seq;
1405     u64 ns;
1406 
1407     userpg->cap_user_time = 0;
1408     userpg->cap_user_time_zero = 0;
1409     userpg->cap_user_time_short = 0;
1410     userpg->cap_user_rdpmc = armv8pmu_event_has_user_read(event);
1411 
1412     if (userpg->cap_user_rdpmc) {
1413         if (event->hw.flags & ARMPMU_EVT_64BIT)
1414             userpg->pmc_width = 64;
1415         else
1416             userpg->pmc_width = 32;
1417     }
1418 
1419     do {
1420         rd = sched_clock_read_begin(&seq);
1421 
1422         if (rd->read_sched_clock != arch_timer_read_counter)
1423             return;
1424 
1425         userpg->time_mult = rd->mult;
1426         userpg->time_shift = rd->shift;
1427         userpg->time_zero = rd->epoch_ns;
1428         userpg->time_cycles = rd->epoch_cyc;
1429         userpg->time_mask = rd->sched_clock_mask;
1430 
1431         /*
1432          * Subtract the cycle base, such that software that
1433          * doesn't know about cap_user_time_short still 'works'
1434          * assuming no wraps.
1435          */
1436         ns = mul_u64_u32_shr(rd->epoch_cyc, rd->mult, rd->shift);
1437         userpg->time_zero -= ns;
1438 
1439     } while (sched_clock_read_retry(seq));
1440 
1441     userpg->time_offset = userpg->time_zero - now;
1442 
1443     /*
1444      * time_shift is not expected to be greater than 31 due to
1445      * the original published conversion algorithm shifting a
1446      * 32-bit value (now specifies a 64-bit value) - refer
1447      * perf_event_mmap_page documentation in perf_event.h.
1448      */
1449     if (userpg->time_shift == 32) {
1450         userpg->time_shift = 31;
1451         userpg->time_mult >>= 1;
1452     }
1453 
1454     /*
1455      * Internal timekeeping for enabled/running/stopped times
1456      * is always computed with the sched_clock.
1457      */
1458     userpg->cap_user_time = 1;
1459     userpg->cap_user_time_zero = 1;
1460     userpg->cap_user_time_short = 1;
1461 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!