Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 // CCI Cache Coherent Interconnect PMU driver
0003 // Copyright (C) 2013-2018 Arm Ltd.
0004 // Author: Punit Agrawal <punit.agrawal@arm.com>, Suzuki Poulose <suzuki.poulose@arm.com>
0005 
0006 #include <linux/arm-cci.h>
0007 #include <linux/io.h>
0008 #include <linux/interrupt.h>
0009 #include <linux/module.h>
0010 #include <linux/of_address.h>
0011 #include <linux/of_device.h>
0012 #include <linux/of_irq.h>
0013 #include <linux/of_platform.h>
0014 #include <linux/perf_event.h>
0015 #include <linux/platform_device.h>
0016 #include <linux/slab.h>
0017 #include <linux/spinlock.h>
0018 
0019 #define DRIVER_NAME     "ARM-CCI PMU"
0020 
0021 #define CCI_PMCR        0x0100
0022 #define CCI_PID2        0x0fe8
0023 
0024 #define CCI_PMCR_CEN        0x00000001
0025 #define CCI_PMCR_NCNT_MASK  0x0000f800
0026 #define CCI_PMCR_NCNT_SHIFT 11
0027 
0028 #define CCI_PID2_REV_MASK   0xf0
0029 #define CCI_PID2_REV_SHIFT  4
0030 
0031 #define CCI_PMU_EVT_SEL     0x000
0032 #define CCI_PMU_CNTR        0x004
0033 #define CCI_PMU_CNTR_CTRL   0x008
0034 #define CCI_PMU_OVRFLW      0x00c
0035 
0036 #define CCI_PMU_OVRFLW_FLAG 1
0037 
0038 #define CCI_PMU_CNTR_SIZE(model)    ((model)->cntr_size)
0039 #define CCI_PMU_CNTR_BASE(model, idx)   ((idx) * CCI_PMU_CNTR_SIZE(model))
0040 #define CCI_PMU_CNTR_MASK       ((1ULL << 32) - 1)
0041 #define CCI_PMU_CNTR_LAST(cci_pmu)  (cci_pmu->num_cntrs - 1)
0042 
0043 #define CCI_PMU_MAX_HW_CNTRS(model) \
0044     ((model)->num_hw_cntrs + (model)->fixed_hw_cntrs)
0045 
0046 /* Types of interfaces that can generate events */
0047 enum {
0048     CCI_IF_SLAVE,
0049     CCI_IF_MASTER,
0050 #ifdef CONFIG_ARM_CCI5xx_PMU
0051     CCI_IF_GLOBAL,
0052 #endif
0053     CCI_IF_MAX,
0054 };
0055 
0056 #define NUM_HW_CNTRS_CII_4XX    4
0057 #define NUM_HW_CNTRS_CII_5XX    8
0058 #define NUM_HW_CNTRS_MAX    NUM_HW_CNTRS_CII_5XX
0059 
0060 #define FIXED_HW_CNTRS_CII_4XX  1
0061 #define FIXED_HW_CNTRS_CII_5XX  0
0062 #define FIXED_HW_CNTRS_MAX  FIXED_HW_CNTRS_CII_4XX
0063 
0064 #define HW_CNTRS_MAX        (NUM_HW_CNTRS_MAX + FIXED_HW_CNTRS_MAX)
0065 
0066 struct event_range {
0067     u32 min;
0068     u32 max;
0069 };
0070 
0071 struct cci_pmu_hw_events {
0072     struct perf_event **events;
0073     unsigned long *used_mask;
0074     raw_spinlock_t pmu_lock;
0075 };
0076 
0077 struct cci_pmu;
0078 /*
0079  * struct cci_pmu_model:
0080  * @fixed_hw_cntrs - Number of fixed event counters
0081  * @num_hw_cntrs - Maximum number of programmable event counters
0082  * @cntr_size - Size of an event counter mapping
0083  */
0084 struct cci_pmu_model {
0085     char *name;
0086     u32 fixed_hw_cntrs;
0087     u32 num_hw_cntrs;
0088     u32 cntr_size;
0089     struct attribute **format_attrs;
0090     struct attribute **event_attrs;
0091     struct event_range event_ranges[CCI_IF_MAX];
0092     int (*validate_hw_event)(struct cci_pmu *, unsigned long);
0093     int (*get_event_idx)(struct cci_pmu *, struct cci_pmu_hw_events *, unsigned long);
0094     void (*write_counters)(struct cci_pmu *, unsigned long *);
0095 };
0096 
0097 static struct cci_pmu_model cci_pmu_models[];
0098 
0099 struct cci_pmu {
0100     void __iomem *base;
0101     void __iomem *ctrl_base;
0102     struct pmu pmu;
0103     int cpu;
0104     int nr_irqs;
0105     int *irqs;
0106     unsigned long active_irqs;
0107     const struct cci_pmu_model *model;
0108     struct cci_pmu_hw_events hw_events;
0109     struct platform_device *plat_device;
0110     int num_cntrs;
0111     atomic_t active_events;
0112     struct mutex reserve_mutex;
0113 };
0114 
0115 #define to_cci_pmu(c)   (container_of(c, struct cci_pmu, pmu))
0116 
0117 static struct cci_pmu *g_cci_pmu;
0118 
0119 enum cci_models {
0120 #ifdef CONFIG_ARM_CCI400_PMU
0121     CCI400_R0,
0122     CCI400_R1,
0123 #endif
0124 #ifdef CONFIG_ARM_CCI5xx_PMU
0125     CCI500_R0,
0126     CCI550_R0,
0127 #endif
0128     CCI_MODEL_MAX
0129 };
0130 
0131 static void pmu_write_counters(struct cci_pmu *cci_pmu,
0132                  unsigned long *mask);
0133 static ssize_t __maybe_unused cci_pmu_format_show(struct device *dev,
0134             struct device_attribute *attr, char *buf);
0135 static ssize_t __maybe_unused cci_pmu_event_show(struct device *dev,
0136             struct device_attribute *attr, char *buf);
0137 
0138 #define CCI_EXT_ATTR_ENTRY(_name, _func, _config)               \
0139     &((struct dev_ext_attribute[]) {                    \
0140         { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config }    \
0141     })[0].attr.attr
0142 
0143 #define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \
0144     CCI_EXT_ATTR_ENTRY(_name, cci_pmu_format_show, (char *)_config)
0145 #define CCI_EVENT_EXT_ATTR_ENTRY(_name, _config) \
0146     CCI_EXT_ATTR_ENTRY(_name, cci_pmu_event_show, (unsigned long)_config)
0147 
0148 /* CCI400 PMU Specific definitions */
0149 
0150 #ifdef CONFIG_ARM_CCI400_PMU
0151 
0152 /* Port ids */
0153 #define CCI400_PORT_S0      0
0154 #define CCI400_PORT_S1      1
0155 #define CCI400_PORT_S2      2
0156 #define CCI400_PORT_S3      3
0157 #define CCI400_PORT_S4      4
0158 #define CCI400_PORT_M0      5
0159 #define CCI400_PORT_M1      6
0160 #define CCI400_PORT_M2      7
0161 
0162 #define CCI400_R1_PX        5
0163 
0164 /*
0165  * Instead of an event id to monitor CCI cycles, a dedicated counter is
0166  * provided. Use 0xff to represent CCI cycles and hope that no future revisions
0167  * make use of this event in hardware.
0168  */
0169 enum cci400_perf_events {
0170     CCI400_PMU_CYCLES = 0xff
0171 };
0172 
0173 #define CCI400_PMU_CYCLE_CNTR_IDX   0
0174 #define CCI400_PMU_CNTR0_IDX        1
0175 
0176 /*
0177  * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8
0178  * ports and bits 4:0 are event codes. There are different event codes
0179  * associated with each port type.
0180  *
0181  * Additionally, the range of events associated with the port types changed
0182  * between Rev0 and Rev1.
0183  *
0184  * The constants below define the range of valid codes for each port type for
0185  * the different revisions and are used to validate the event to be monitored.
0186  */
0187 
0188 #define CCI400_PMU_EVENT_MASK       0xffUL
0189 #define CCI400_PMU_EVENT_SOURCE_SHIFT   5
0190 #define CCI400_PMU_EVENT_SOURCE_MASK    0x7
0191 #define CCI400_PMU_EVENT_CODE_SHIFT 0
0192 #define CCI400_PMU_EVENT_CODE_MASK  0x1f
0193 #define CCI400_PMU_EVENT_SOURCE(event) \
0194     ((event >> CCI400_PMU_EVENT_SOURCE_SHIFT) & \
0195             CCI400_PMU_EVENT_SOURCE_MASK)
0196 #define CCI400_PMU_EVENT_CODE(event) \
0197     ((event >> CCI400_PMU_EVENT_CODE_SHIFT) & CCI400_PMU_EVENT_CODE_MASK)
0198 
0199 #define CCI400_R0_SLAVE_PORT_MIN_EV 0x00
0200 #define CCI400_R0_SLAVE_PORT_MAX_EV 0x13
0201 #define CCI400_R0_MASTER_PORT_MIN_EV    0x14
0202 #define CCI400_R0_MASTER_PORT_MAX_EV    0x1a
0203 
0204 #define CCI400_R1_SLAVE_PORT_MIN_EV 0x00
0205 #define CCI400_R1_SLAVE_PORT_MAX_EV 0x14
0206 #define CCI400_R1_MASTER_PORT_MIN_EV    0x00
0207 #define CCI400_R1_MASTER_PORT_MAX_EV    0x11
0208 
0209 #define CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(_name, _config) \
0210     CCI_EXT_ATTR_ENTRY(_name, cci400_pmu_cycle_event_show, \
0211                     (unsigned long)_config)
0212 
0213 static ssize_t cci400_pmu_cycle_event_show(struct device *dev,
0214             struct device_attribute *attr, char *buf);
0215 
0216 static struct attribute *cci400_pmu_format_attrs[] = {
0217     CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),
0218     CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-7"),
0219     NULL
0220 };
0221 
0222 static struct attribute *cci400_r0_pmu_event_attrs[] = {
0223     /* Slave events */
0224     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),
0225     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),
0226     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),
0227     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),
0228     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),
0229     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),
0230     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),
0231     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
0232     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),
0233     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),
0234     CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),
0235     CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),
0236     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),
0237     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),
0238     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),
0239     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),
0240     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),
0241     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),
0242     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),
0243     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),
0244     /* Master events */
0245     CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x14),
0246     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_addr_hazard, 0x15),
0247     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_id_hazard, 0x16),
0248     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_tt_full, 0x17),
0249     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x18),
0250     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x19),
0251     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_tt_full, 0x1A),
0252     /* Special event for cycles counter */
0253     CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),
0254     NULL
0255 };
0256 
0257 static struct attribute *cci400_r1_pmu_event_attrs[] = {
0258     /* Slave events */
0259     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),
0260     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),
0261     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),
0262     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),
0263     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),
0264     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),
0265     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),
0266     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
0267     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),
0268     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),
0269     CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),
0270     CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),
0271     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),
0272     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),
0273     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),
0274     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),
0275     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),
0276     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),
0277     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),
0278     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),
0279     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_slave_id_hazard, 0x14),
0280     /* Master events */
0281     CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x0),
0282     CCI_EVENT_EXT_ATTR_ENTRY(mi_stall_cycle_addr_hazard, 0x1),
0283     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_master_id_hazard, 0x2),
0284     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_hi_prio_rtq_full, 0x3),
0285     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x4),
0286     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x5),
0287     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_wtq_full, 0x6),
0288     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_low_prio_rtq_full, 0x7),
0289     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_mid_prio_rtq_full, 0x8),
0290     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn0, 0x9),
0291     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn1, 0xA),
0292     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn2, 0xB),
0293     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn3, 0xC),
0294     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn0, 0xD),
0295     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn1, 0xE),
0296     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn2, 0xF),
0297     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn3, 0x10),
0298     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_unique_or_line_unique_addr_hazard, 0x11),
0299     /* Special event for cycles counter */
0300     CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),
0301     NULL
0302 };
0303 
0304 static ssize_t cci400_pmu_cycle_event_show(struct device *dev,
0305             struct device_attribute *attr, char *buf)
0306 {
0307     struct dev_ext_attribute *eattr = container_of(attr,
0308                 struct dev_ext_attribute, attr);
0309     return sysfs_emit(buf, "config=0x%lx\n", (unsigned long)eattr->var);
0310 }
0311 
0312 static int cci400_get_event_idx(struct cci_pmu *cci_pmu,
0313                 struct cci_pmu_hw_events *hw,
0314                 unsigned long cci_event)
0315 {
0316     int idx;
0317 
0318     /* cycles event idx is fixed */
0319     if (cci_event == CCI400_PMU_CYCLES) {
0320         if (test_and_set_bit(CCI400_PMU_CYCLE_CNTR_IDX, hw->used_mask))
0321             return -EAGAIN;
0322 
0323         return CCI400_PMU_CYCLE_CNTR_IDX;
0324     }
0325 
0326     for (idx = CCI400_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx)
0327         if (!test_and_set_bit(idx, hw->used_mask))
0328             return idx;
0329 
0330     /* No counters available */
0331     return -EAGAIN;
0332 }
0333 
0334 static int cci400_validate_hw_event(struct cci_pmu *cci_pmu, unsigned long hw_event)
0335 {
0336     u8 ev_source = CCI400_PMU_EVENT_SOURCE(hw_event);
0337     u8 ev_code = CCI400_PMU_EVENT_CODE(hw_event);
0338     int if_type;
0339 
0340     if (hw_event & ~CCI400_PMU_EVENT_MASK)
0341         return -ENOENT;
0342 
0343     if (hw_event == CCI400_PMU_CYCLES)
0344         return hw_event;
0345 
0346     switch (ev_source) {
0347     case CCI400_PORT_S0:
0348     case CCI400_PORT_S1:
0349     case CCI400_PORT_S2:
0350     case CCI400_PORT_S3:
0351     case CCI400_PORT_S4:
0352         /* Slave Interface */
0353         if_type = CCI_IF_SLAVE;
0354         break;
0355     case CCI400_PORT_M0:
0356     case CCI400_PORT_M1:
0357     case CCI400_PORT_M2:
0358         /* Master Interface */
0359         if_type = CCI_IF_MASTER;
0360         break;
0361     default:
0362         return -ENOENT;
0363     }
0364 
0365     if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
0366         ev_code <= cci_pmu->model->event_ranges[if_type].max)
0367         return hw_event;
0368 
0369     return -ENOENT;
0370 }
0371 
0372 static int probe_cci400_revision(struct cci_pmu *cci_pmu)
0373 {
0374     int rev;
0375     rev = readl_relaxed(cci_pmu->ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK;
0376     rev >>= CCI_PID2_REV_SHIFT;
0377 
0378     if (rev < CCI400_R1_PX)
0379         return CCI400_R0;
0380     else
0381         return CCI400_R1;
0382 }
0383 
0384 static const struct cci_pmu_model *probe_cci_model(struct cci_pmu *cci_pmu)
0385 {
0386     if (platform_has_secure_cci_access())
0387         return &cci_pmu_models[probe_cci400_revision(cci_pmu)];
0388     return NULL;
0389 }
0390 #else   /* !CONFIG_ARM_CCI400_PMU */
0391 static inline struct cci_pmu_model *probe_cci_model(struct cci_pmu *cci_pmu)
0392 {
0393     return NULL;
0394 }
0395 #endif  /* CONFIG_ARM_CCI400_PMU */
0396 
0397 #ifdef CONFIG_ARM_CCI5xx_PMU
0398 
0399 /*
0400  * CCI5xx PMU event id is an 9-bit value made of two parts.
0401  *   bits [8:5] - Source for the event
0402  *   bits [4:0] - Event code (specific to type of interface)
0403  *
0404  *
0405  */
0406 
0407 /* Port ids */
0408 #define CCI5xx_PORT_S0          0x0
0409 #define CCI5xx_PORT_S1          0x1
0410 #define CCI5xx_PORT_S2          0x2
0411 #define CCI5xx_PORT_S3          0x3
0412 #define CCI5xx_PORT_S4          0x4
0413 #define CCI5xx_PORT_S5          0x5
0414 #define CCI5xx_PORT_S6          0x6
0415 
0416 #define CCI5xx_PORT_M0          0x8
0417 #define CCI5xx_PORT_M1          0x9
0418 #define CCI5xx_PORT_M2          0xa
0419 #define CCI5xx_PORT_M3          0xb
0420 #define CCI5xx_PORT_M4          0xc
0421 #define CCI5xx_PORT_M5          0xd
0422 #define CCI5xx_PORT_M6          0xe
0423 
0424 #define CCI5xx_PORT_GLOBAL      0xf
0425 
0426 #define CCI5xx_PMU_EVENT_MASK       0x1ffUL
0427 #define CCI5xx_PMU_EVENT_SOURCE_SHIFT   0x5
0428 #define CCI5xx_PMU_EVENT_SOURCE_MASK    0xf
0429 #define CCI5xx_PMU_EVENT_CODE_SHIFT 0x0
0430 #define CCI5xx_PMU_EVENT_CODE_MASK  0x1f
0431 
0432 #define CCI5xx_PMU_EVENT_SOURCE(event)  \
0433     ((event >> CCI5xx_PMU_EVENT_SOURCE_SHIFT) & CCI5xx_PMU_EVENT_SOURCE_MASK)
0434 #define CCI5xx_PMU_EVENT_CODE(event)    \
0435     ((event >> CCI5xx_PMU_EVENT_CODE_SHIFT) & CCI5xx_PMU_EVENT_CODE_MASK)
0436 
0437 #define CCI5xx_SLAVE_PORT_MIN_EV    0x00
0438 #define CCI5xx_SLAVE_PORT_MAX_EV    0x1f
0439 #define CCI5xx_MASTER_PORT_MIN_EV   0x00
0440 #define CCI5xx_MASTER_PORT_MAX_EV   0x06
0441 #define CCI5xx_GLOBAL_PORT_MIN_EV   0x00
0442 #define CCI5xx_GLOBAL_PORT_MAX_EV   0x0f
0443 
0444 
0445 #define CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(_name, _config) \
0446     CCI_EXT_ATTR_ENTRY(_name, cci5xx_pmu_global_event_show, \
0447                     (unsigned long) _config)
0448 
0449 static ssize_t cci5xx_pmu_global_event_show(struct device *dev,
0450                 struct device_attribute *attr, char *buf);
0451 
0452 static struct attribute *cci5xx_pmu_format_attrs[] = {
0453     CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),
0454     CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-8"),
0455     NULL,
0456 };
0457 
0458 static struct attribute *cci5xx_pmu_event_attrs[] = {
0459     /* Slave events */
0460     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_arvalid, 0x0),
0461     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_dev, 0x1),
0462     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_nonshareable, 0x2),
0463     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_non_alloc, 0x3),
0464     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_alloc, 0x4),
0465     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_invalidate, 0x5),
0466     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maint, 0x6),
0467     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
0468     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rval, 0x8),
0469     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rlast_snoop, 0x9),
0470     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_awalid, 0xA),
0471     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_dev, 0xB),
0472     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_non_shareable, 0xC),
0473     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wb, 0xD),
0474     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wlu, 0xE),
0475     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wunique, 0xF),
0476     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_evict, 0x10),
0477     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_wrevict, 0x11),
0478     CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_beat, 0x12),
0479     CCI_EVENT_EXT_ATTR_ENTRY(si_srq_acvalid, 0x13),
0480     CCI_EVENT_EXT_ATTR_ENTRY(si_srq_read, 0x14),
0481     CCI_EVENT_EXT_ATTR_ENTRY(si_srq_clean, 0x15),
0482     CCI_EVENT_EXT_ATTR_ENTRY(si_srq_data_transfer_low, 0x16),
0483     CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_arvalid, 0x17),
0484     CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall, 0x18),
0485     CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall, 0x19),
0486     CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_stall, 0x1A),
0487     CCI_EVENT_EXT_ATTR_ENTRY(si_w_resp_stall, 0x1B),
0488     CCI_EVENT_EXT_ATTR_ENTRY(si_srq_stall, 0x1C),
0489     CCI_EVENT_EXT_ATTR_ENTRY(si_s_data_stall, 0x1D),
0490     CCI_EVENT_EXT_ATTR_ENTRY(si_rq_stall_ot_limit, 0x1E),
0491     CCI_EVENT_EXT_ATTR_ENTRY(si_r_stall_arbit, 0x1F),
0492 
0493     /* Master events */
0494     CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_beat_any, 0x0),
0495     CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_beat_any, 0x1),
0496     CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall, 0x2),
0497     CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_stall, 0x3),
0498     CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall, 0x4),
0499     CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_stall, 0x5),
0500     CCI_EVENT_EXT_ATTR_ENTRY(mi_w_resp_stall, 0x6),
0501 
0502     /* Global events */
0503     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_0_1, 0x0),
0504     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_2_3, 0x1),
0505     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_4_5, 0x2),
0506     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_6_7, 0x3),
0507     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_0_1, 0x4),
0508     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_2_3, 0x5),
0509     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_4_5, 0x6),
0510     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_6_7, 0x7),
0511     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_back_invalidation, 0x8),
0512     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_alloc_busy, 0x9),
0513     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_tt_full, 0xA),
0514     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB),
0515     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC),
0516     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD),
0517     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_stall_tt_full, 0xE),
0518     CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF),
0519     NULL
0520 };
0521 
0522 static ssize_t cci5xx_pmu_global_event_show(struct device *dev,
0523                 struct device_attribute *attr, char *buf)
0524 {
0525     struct dev_ext_attribute *eattr = container_of(attr,
0526                     struct dev_ext_attribute, attr);
0527     /* Global events have single fixed source code */
0528     return sysfs_emit(buf, "event=0x%lx,source=0x%x\n",
0529               (unsigned long)eattr->var, CCI5xx_PORT_GLOBAL);
0530 }
0531 
0532 /*
0533  * CCI500 provides 8 independent event counters that can count
0534  * any of the events available.
0535  * CCI500 PMU event source ids
0536  *  0x0-0x6 - Slave interfaces
0537  *  0x8-0xD - Master interfaces
0538  *  0xf     - Global Events
0539  *  0x7,0xe - Reserved
0540  */
0541 static int cci500_validate_hw_event(struct cci_pmu *cci_pmu,
0542                     unsigned long hw_event)
0543 {
0544     u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);
0545     u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);
0546     int if_type;
0547 
0548     if (hw_event & ~CCI5xx_PMU_EVENT_MASK)
0549         return -ENOENT;
0550 
0551     switch (ev_source) {
0552     case CCI5xx_PORT_S0:
0553     case CCI5xx_PORT_S1:
0554     case CCI5xx_PORT_S2:
0555     case CCI5xx_PORT_S3:
0556     case CCI5xx_PORT_S4:
0557     case CCI5xx_PORT_S5:
0558     case CCI5xx_PORT_S6:
0559         if_type = CCI_IF_SLAVE;
0560         break;
0561     case CCI5xx_PORT_M0:
0562     case CCI5xx_PORT_M1:
0563     case CCI5xx_PORT_M2:
0564     case CCI5xx_PORT_M3:
0565     case CCI5xx_PORT_M4:
0566     case CCI5xx_PORT_M5:
0567         if_type = CCI_IF_MASTER;
0568         break;
0569     case CCI5xx_PORT_GLOBAL:
0570         if_type = CCI_IF_GLOBAL;
0571         break;
0572     default:
0573         return -ENOENT;
0574     }
0575 
0576     if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
0577         ev_code <= cci_pmu->model->event_ranges[if_type].max)
0578         return hw_event;
0579 
0580     return -ENOENT;
0581 }
0582 
0583 /*
0584  * CCI550 provides 8 independent event counters that can count
0585  * any of the events available.
0586  * CCI550 PMU event source ids
0587  *  0x0-0x6 - Slave interfaces
0588  *  0x8-0xe - Master interfaces
0589  *  0xf     - Global Events
0590  *  0x7 - Reserved
0591  */
0592 static int cci550_validate_hw_event(struct cci_pmu *cci_pmu,
0593                     unsigned long hw_event)
0594 {
0595     u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);
0596     u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);
0597     int if_type;
0598 
0599     if (hw_event & ~CCI5xx_PMU_EVENT_MASK)
0600         return -ENOENT;
0601 
0602     switch (ev_source) {
0603     case CCI5xx_PORT_S0:
0604     case CCI5xx_PORT_S1:
0605     case CCI5xx_PORT_S2:
0606     case CCI5xx_PORT_S3:
0607     case CCI5xx_PORT_S4:
0608     case CCI5xx_PORT_S5:
0609     case CCI5xx_PORT_S6:
0610         if_type = CCI_IF_SLAVE;
0611         break;
0612     case CCI5xx_PORT_M0:
0613     case CCI5xx_PORT_M1:
0614     case CCI5xx_PORT_M2:
0615     case CCI5xx_PORT_M3:
0616     case CCI5xx_PORT_M4:
0617     case CCI5xx_PORT_M5:
0618     case CCI5xx_PORT_M6:
0619         if_type = CCI_IF_MASTER;
0620         break;
0621     case CCI5xx_PORT_GLOBAL:
0622         if_type = CCI_IF_GLOBAL;
0623         break;
0624     default:
0625         return -ENOENT;
0626     }
0627 
0628     if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
0629         ev_code <= cci_pmu->model->event_ranges[if_type].max)
0630         return hw_event;
0631 
0632     return -ENOENT;
0633 }
0634 
0635 #endif  /* CONFIG_ARM_CCI5xx_PMU */
0636 
0637 /*
0638  * Program the CCI PMU counters which have PERF_HES_ARCH set
0639  * with the event period and mark them ready before we enable
0640  * PMU.
0641  */
0642 static void cci_pmu_sync_counters(struct cci_pmu *cci_pmu)
0643 {
0644     int i;
0645     struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;
0646     DECLARE_BITMAP(mask, HW_CNTRS_MAX);
0647 
0648     bitmap_zero(mask, cci_pmu->num_cntrs);
0649     for_each_set_bit(i, cci_pmu->hw_events.used_mask, cci_pmu->num_cntrs) {
0650         struct perf_event *event = cci_hw->events[i];
0651 
0652         if (WARN_ON(!event))
0653             continue;
0654 
0655         /* Leave the events which are not counting */
0656         if (event->hw.state & PERF_HES_STOPPED)
0657             continue;
0658         if (event->hw.state & PERF_HES_ARCH) {
0659             set_bit(i, mask);
0660             event->hw.state &= ~PERF_HES_ARCH;
0661         }
0662     }
0663 
0664     pmu_write_counters(cci_pmu, mask);
0665 }
0666 
0667 /* Should be called with cci_pmu->hw_events->pmu_lock held */
0668 static void __cci_pmu_enable_nosync(struct cci_pmu *cci_pmu)
0669 {
0670     u32 val;
0671 
0672     /* Enable all the PMU counters. */
0673     val = readl_relaxed(cci_pmu->ctrl_base + CCI_PMCR) | CCI_PMCR_CEN;
0674     writel(val, cci_pmu->ctrl_base + CCI_PMCR);
0675 }
0676 
0677 /* Should be called with cci_pmu->hw_events->pmu_lock held */
0678 static void __cci_pmu_enable_sync(struct cci_pmu *cci_pmu)
0679 {
0680     cci_pmu_sync_counters(cci_pmu);
0681     __cci_pmu_enable_nosync(cci_pmu);
0682 }
0683 
0684 /* Should be called with cci_pmu->hw_events->pmu_lock held */
0685 static void __cci_pmu_disable(struct cci_pmu *cci_pmu)
0686 {
0687     u32 val;
0688 
0689     /* Disable all the PMU counters. */
0690     val = readl_relaxed(cci_pmu->ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN;
0691     writel(val, cci_pmu->ctrl_base + CCI_PMCR);
0692 }
0693 
0694 static ssize_t cci_pmu_format_show(struct device *dev,
0695             struct device_attribute *attr, char *buf)
0696 {
0697     struct dev_ext_attribute *eattr = container_of(attr,
0698                 struct dev_ext_attribute, attr);
0699     return sysfs_emit(buf, "%s\n", (char *)eattr->var);
0700 }
0701 
0702 static ssize_t cci_pmu_event_show(struct device *dev,
0703             struct device_attribute *attr, char *buf)
0704 {
0705     struct dev_ext_attribute *eattr = container_of(attr,
0706                 struct dev_ext_attribute, attr);
0707     /* source parameter is mandatory for normal PMU events */
0708     return sysfs_emit(buf, "source=?,event=0x%lx\n",
0709               (unsigned long)eattr->var);
0710 }
0711 
0712 static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx)
0713 {
0714     return 0 <= idx && idx <= CCI_PMU_CNTR_LAST(cci_pmu);
0715 }
0716 
0717 static u32 pmu_read_register(struct cci_pmu *cci_pmu, int idx, unsigned int offset)
0718 {
0719     return readl_relaxed(cci_pmu->base +
0720                  CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);
0721 }
0722 
0723 static void pmu_write_register(struct cci_pmu *cci_pmu, u32 value,
0724                    int idx, unsigned int offset)
0725 {
0726     writel_relaxed(value, cci_pmu->base +
0727                CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);
0728 }
0729 
0730 static void pmu_disable_counter(struct cci_pmu *cci_pmu, int idx)
0731 {
0732     pmu_write_register(cci_pmu, 0, idx, CCI_PMU_CNTR_CTRL);
0733 }
0734 
0735 static void pmu_enable_counter(struct cci_pmu *cci_pmu, int idx)
0736 {
0737     pmu_write_register(cci_pmu, 1, idx, CCI_PMU_CNTR_CTRL);
0738 }
0739 
0740 static bool __maybe_unused
0741 pmu_counter_is_enabled(struct cci_pmu *cci_pmu, int idx)
0742 {
0743     return (pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR_CTRL) & 0x1) != 0;
0744 }
0745 
0746 static void pmu_set_event(struct cci_pmu *cci_pmu, int idx, unsigned long event)
0747 {
0748     pmu_write_register(cci_pmu, event, idx, CCI_PMU_EVT_SEL);
0749 }
0750 
0751 /*
0752  * For all counters on the CCI-PMU, disable any 'enabled' counters,
0753  * saving the changed counters in the mask, so that we can restore
0754  * it later using pmu_restore_counters. The mask is private to the
0755  * caller. We cannot rely on the used_mask maintained by the CCI_PMU
0756  * as it only tells us if the counter is assigned to perf_event or not.
0757  * The state of the perf_event cannot be locked by the PMU layer, hence
0758  * we check the individual counter status (which can be locked by
0759  * cci_pm->hw_events->pmu_lock).
0760  *
0761  * @mask should be initialised to empty by the caller.
0762  */
0763 static void __maybe_unused
0764 pmu_save_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
0765 {
0766     int i;
0767 
0768     for (i = 0; i < cci_pmu->num_cntrs; i++) {
0769         if (pmu_counter_is_enabled(cci_pmu, i)) {
0770             set_bit(i, mask);
0771             pmu_disable_counter(cci_pmu, i);
0772         }
0773     }
0774 }
0775 
0776 /*
0777  * Restore the status of the counters. Reversal of the pmu_save_counters().
0778  * For each counter set in the mask, enable the counter back.
0779  */
0780 static void __maybe_unused
0781 pmu_restore_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
0782 {
0783     int i;
0784 
0785     for_each_set_bit(i, mask, cci_pmu->num_cntrs)
0786         pmu_enable_counter(cci_pmu, i);
0787 }
0788 
0789 /*
0790  * Returns the number of programmable counters actually implemented
0791  * by the cci
0792  */
0793 static u32 pmu_get_max_counters(struct cci_pmu *cci_pmu)
0794 {
0795     return (readl_relaxed(cci_pmu->ctrl_base + CCI_PMCR) &
0796         CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT;
0797 }
0798 
0799 static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *event)
0800 {
0801     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
0802     unsigned long cci_event = event->hw.config_base;
0803     int idx;
0804 
0805     if (cci_pmu->model->get_event_idx)
0806         return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event);
0807 
0808     /* Generic code to find an unused idx from the mask */
0809     for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++)
0810         if (!test_and_set_bit(idx, hw->used_mask))
0811             return idx;
0812 
0813     /* No counters available */
0814     return -EAGAIN;
0815 }
0816 
0817 static int pmu_map_event(struct perf_event *event)
0818 {
0819     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
0820 
0821     if (event->attr.type < PERF_TYPE_MAX ||
0822             !cci_pmu->model->validate_hw_event)
0823         return -ENOENT;
0824 
0825     return  cci_pmu->model->validate_hw_event(cci_pmu, event->attr.config);
0826 }
0827 
0828 static int pmu_request_irq(struct cci_pmu *cci_pmu, irq_handler_t handler)
0829 {
0830     int i;
0831     struct platform_device *pmu_device = cci_pmu->plat_device;
0832 
0833     if (unlikely(!pmu_device))
0834         return -ENODEV;
0835 
0836     if (cci_pmu->nr_irqs < 1) {
0837         dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n");
0838         return -ENODEV;
0839     }
0840 
0841     /*
0842      * Register all available CCI PMU interrupts. In the interrupt handler
0843      * we iterate over the counters checking for interrupt source (the
0844      * overflowing counter) and clear it.
0845      *
0846      * This should allow handling of non-unique interrupt for the counters.
0847      */
0848     for (i = 0; i < cci_pmu->nr_irqs; i++) {
0849         int err = request_irq(cci_pmu->irqs[i], handler, IRQF_SHARED,
0850                 "arm-cci-pmu", cci_pmu);
0851         if (err) {
0852             dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n",
0853                 cci_pmu->irqs[i]);
0854             return err;
0855         }
0856 
0857         set_bit(i, &cci_pmu->active_irqs);
0858     }
0859 
0860     return 0;
0861 }
0862 
0863 static void pmu_free_irq(struct cci_pmu *cci_pmu)
0864 {
0865     int i;
0866 
0867     for (i = 0; i < cci_pmu->nr_irqs; i++) {
0868         if (!test_and_clear_bit(i, &cci_pmu->active_irqs))
0869             continue;
0870 
0871         free_irq(cci_pmu->irqs[i], cci_pmu);
0872     }
0873 }
0874 
0875 static u32 pmu_read_counter(struct perf_event *event)
0876 {
0877     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
0878     struct hw_perf_event *hw_counter = &event->hw;
0879     int idx = hw_counter->idx;
0880     u32 value;
0881 
0882     if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
0883         dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
0884         return 0;
0885     }
0886     value = pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR);
0887 
0888     return value;
0889 }
0890 
0891 static void pmu_write_counter(struct cci_pmu *cci_pmu, u32 value, int idx)
0892 {
0893     pmu_write_register(cci_pmu, value, idx, CCI_PMU_CNTR);
0894 }
0895 
0896 static void __pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
0897 {
0898     int i;
0899     struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;
0900 
0901     for_each_set_bit(i, mask, cci_pmu->num_cntrs) {
0902         struct perf_event *event = cci_hw->events[i];
0903 
0904         if (WARN_ON(!event))
0905             continue;
0906         pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);
0907     }
0908 }
0909 
0910 static void pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
0911 {
0912     if (cci_pmu->model->write_counters)
0913         cci_pmu->model->write_counters(cci_pmu, mask);
0914     else
0915         __pmu_write_counters(cci_pmu, mask);
0916 }
0917 
0918 #ifdef CONFIG_ARM_CCI5xx_PMU
0919 
0920 /*
0921  * CCI-500/CCI-550 has advanced power saving policies, which could gate the
0922  * clocks to the PMU counters, which makes the writes to them ineffective.
0923  * The only way to write to those counters is when the global counters
0924  * are enabled and the particular counter is enabled.
0925  *
0926  * So we do the following :
0927  *
0928  * 1) Disable all the PMU counters, saving their current state
0929  * 2) Enable the global PMU profiling, now that all counters are
0930  *    disabled.
0931  *
0932  * For each counter to be programmed, repeat steps 3-7:
0933  *
0934  * 3) Write an invalid event code to the event control register for the
0935       counter, so that the counters are not modified.
0936  * 4) Enable the counter control for the counter.
0937  * 5) Set the counter value
0938  * 6) Disable the counter
0939  * 7) Restore the event in the target counter
0940  *
0941  * 8) Disable the global PMU.
0942  * 9) Restore the status of the rest of the counters.
0943  *
0944  * We choose an event which for CCI-5xx is guaranteed not to count.
0945  * We use the highest possible event code (0x1f) for the master interface 0.
0946  */
0947 #define CCI5xx_INVALID_EVENT    ((CCI5xx_PORT_M0 << CCI5xx_PMU_EVENT_SOURCE_SHIFT) | \
0948                  (CCI5xx_PMU_EVENT_CODE_MASK << CCI5xx_PMU_EVENT_CODE_SHIFT))
0949 static void cci5xx_pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
0950 {
0951     int i;
0952     DECLARE_BITMAP(saved_mask, HW_CNTRS_MAX);
0953 
0954     bitmap_zero(saved_mask, cci_pmu->num_cntrs);
0955     pmu_save_counters(cci_pmu, saved_mask);
0956 
0957     /*
0958      * Now that all the counters are disabled, we can safely turn the PMU on,
0959      * without syncing the status of the counters
0960      */
0961     __cci_pmu_enable_nosync(cci_pmu);
0962 
0963     for_each_set_bit(i, mask, cci_pmu->num_cntrs) {
0964         struct perf_event *event = cci_pmu->hw_events.events[i];
0965 
0966         if (WARN_ON(!event))
0967             continue;
0968 
0969         pmu_set_event(cci_pmu, i, CCI5xx_INVALID_EVENT);
0970         pmu_enable_counter(cci_pmu, i);
0971         pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);
0972         pmu_disable_counter(cci_pmu, i);
0973         pmu_set_event(cci_pmu, i, event->hw.config_base);
0974     }
0975 
0976     __cci_pmu_disable(cci_pmu);
0977 
0978     pmu_restore_counters(cci_pmu, saved_mask);
0979 }
0980 
0981 #endif  /* CONFIG_ARM_CCI5xx_PMU */
0982 
0983 static u64 pmu_event_update(struct perf_event *event)
0984 {
0985     struct hw_perf_event *hwc = &event->hw;
0986     u64 delta, prev_raw_count, new_raw_count;
0987 
0988     do {
0989         prev_raw_count = local64_read(&hwc->prev_count);
0990         new_raw_count = pmu_read_counter(event);
0991     } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
0992          new_raw_count) != prev_raw_count);
0993 
0994     delta = (new_raw_count - prev_raw_count) & CCI_PMU_CNTR_MASK;
0995 
0996     local64_add(delta, &event->count);
0997 
0998     return new_raw_count;
0999 }
1000 
1001 static void pmu_read(struct perf_event *event)
1002 {
1003     pmu_event_update(event);
1004 }
1005 
1006 static void pmu_event_set_period(struct perf_event *event)
1007 {
1008     struct hw_perf_event *hwc = &event->hw;
1009     /*
1010      * The CCI PMU counters have a period of 2^32. To account for the
1011      * possiblity of extreme interrupt latency we program for a period of
1012      * half that. Hopefully we can handle the interrupt before another 2^31
1013      * events occur and the counter overtakes its previous value.
1014      */
1015     u64 val = 1ULL << 31;
1016     local64_set(&hwc->prev_count, val);
1017 
1018     /*
1019      * CCI PMU uses PERF_HES_ARCH to keep track of the counters, whose
1020      * values needs to be sync-ed with the s/w state before the PMU is
1021      * enabled.
1022      * Mark this counter for sync.
1023      */
1024     hwc->state |= PERF_HES_ARCH;
1025 }
1026 
1027 static irqreturn_t pmu_handle_irq(int irq_num, void *dev)
1028 {
1029     struct cci_pmu *cci_pmu = dev;
1030     struct cci_pmu_hw_events *events = &cci_pmu->hw_events;
1031     int idx, handled = IRQ_NONE;
1032 
1033     raw_spin_lock(&events->pmu_lock);
1034 
1035     /* Disable the PMU while we walk through the counters */
1036     __cci_pmu_disable(cci_pmu);
1037     /*
1038      * Iterate over counters and update the corresponding perf events.
1039      * This should work regardless of whether we have per-counter overflow
1040      * interrupt or a combined overflow interrupt.
1041      */
1042     for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) {
1043         struct perf_event *event = events->events[idx];
1044 
1045         if (!event)
1046             continue;
1047 
1048         /* Did this counter overflow? */
1049         if (!(pmu_read_register(cci_pmu, idx, CCI_PMU_OVRFLW) &
1050               CCI_PMU_OVRFLW_FLAG))
1051             continue;
1052 
1053         pmu_write_register(cci_pmu, CCI_PMU_OVRFLW_FLAG, idx,
1054                             CCI_PMU_OVRFLW);
1055 
1056         pmu_event_update(event);
1057         pmu_event_set_period(event);
1058         handled = IRQ_HANDLED;
1059     }
1060 
1061     /* Enable the PMU and sync possibly overflowed counters */
1062     __cci_pmu_enable_sync(cci_pmu);
1063     raw_spin_unlock(&events->pmu_lock);
1064 
1065     return IRQ_RETVAL(handled);
1066 }
1067 
1068 static int cci_pmu_get_hw(struct cci_pmu *cci_pmu)
1069 {
1070     int ret = pmu_request_irq(cci_pmu, pmu_handle_irq);
1071     if (ret) {
1072         pmu_free_irq(cci_pmu);
1073         return ret;
1074     }
1075     return 0;
1076 }
1077 
1078 static void cci_pmu_put_hw(struct cci_pmu *cci_pmu)
1079 {
1080     pmu_free_irq(cci_pmu);
1081 }
1082 
1083 static void hw_perf_event_destroy(struct perf_event *event)
1084 {
1085     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1086     atomic_t *active_events = &cci_pmu->active_events;
1087     struct mutex *reserve_mutex = &cci_pmu->reserve_mutex;
1088 
1089     if (atomic_dec_and_mutex_lock(active_events, reserve_mutex)) {
1090         cci_pmu_put_hw(cci_pmu);
1091         mutex_unlock(reserve_mutex);
1092     }
1093 }
1094 
1095 static void cci_pmu_enable(struct pmu *pmu)
1096 {
1097     struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1098     struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1099     bool enabled = !bitmap_empty(hw_events->used_mask, cci_pmu->num_cntrs);
1100     unsigned long flags;
1101 
1102     if (!enabled)
1103         return;
1104 
1105     raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1106     __cci_pmu_enable_sync(cci_pmu);
1107     raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1108 
1109 }
1110 
1111 static void cci_pmu_disable(struct pmu *pmu)
1112 {
1113     struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1114     struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1115     unsigned long flags;
1116 
1117     raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1118     __cci_pmu_disable(cci_pmu);
1119     raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1120 }
1121 
1122 /*
1123  * Check if the idx represents a non-programmable counter.
1124  * All the fixed event counters are mapped before the programmable
1125  * counters.
1126  */
1127 static bool pmu_fixed_hw_idx(struct cci_pmu *cci_pmu, int idx)
1128 {
1129     return (idx >= 0) && (idx < cci_pmu->model->fixed_hw_cntrs);
1130 }
1131 
1132 static void cci_pmu_start(struct perf_event *event, int pmu_flags)
1133 {
1134     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1135     struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1136     struct hw_perf_event *hwc = &event->hw;
1137     int idx = hwc->idx;
1138     unsigned long flags;
1139 
1140     /*
1141      * To handle interrupt latency, we always reprogram the period
1142      * regardless of PERF_EF_RELOAD.
1143      */
1144     if (pmu_flags & PERF_EF_RELOAD)
1145         WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
1146 
1147     hwc->state = 0;
1148 
1149     if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
1150         dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
1151         return;
1152     }
1153 
1154     raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1155 
1156     /* Configure the counter unless you are counting a fixed event */
1157     if (!pmu_fixed_hw_idx(cci_pmu, idx))
1158         pmu_set_event(cci_pmu, idx, hwc->config_base);
1159 
1160     pmu_event_set_period(event);
1161     pmu_enable_counter(cci_pmu, idx);
1162 
1163     raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1164 }
1165 
1166 static void cci_pmu_stop(struct perf_event *event, int pmu_flags)
1167 {
1168     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1169     struct hw_perf_event *hwc = &event->hw;
1170     int idx = hwc->idx;
1171 
1172     if (hwc->state & PERF_HES_STOPPED)
1173         return;
1174 
1175     if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
1176         dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
1177         return;
1178     }
1179 
1180     /*
1181      * We always reprogram the counter, so ignore PERF_EF_UPDATE. See
1182      * cci_pmu_start()
1183      */
1184     pmu_disable_counter(cci_pmu, idx);
1185     pmu_event_update(event);
1186     hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
1187 }
1188 
1189 static int cci_pmu_add(struct perf_event *event, int flags)
1190 {
1191     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1192     struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1193     struct hw_perf_event *hwc = &event->hw;
1194     int idx;
1195 
1196     /* If we don't have a space for the counter then finish early. */
1197     idx = pmu_get_event_idx(hw_events, event);
1198     if (idx < 0)
1199         return idx;
1200 
1201     event->hw.idx = idx;
1202     hw_events->events[idx] = event;
1203 
1204     hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1205     if (flags & PERF_EF_START)
1206         cci_pmu_start(event, PERF_EF_RELOAD);
1207 
1208     /* Propagate our changes to the userspace mapping. */
1209     perf_event_update_userpage(event);
1210 
1211     return 0;
1212 }
1213 
1214 static void cci_pmu_del(struct perf_event *event, int flags)
1215 {
1216     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1217     struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1218     struct hw_perf_event *hwc = &event->hw;
1219     int idx = hwc->idx;
1220 
1221     cci_pmu_stop(event, PERF_EF_UPDATE);
1222     hw_events->events[idx] = NULL;
1223     clear_bit(idx, hw_events->used_mask);
1224 
1225     perf_event_update_userpage(event);
1226 }
1227 
1228 static int validate_event(struct pmu *cci_pmu,
1229               struct cci_pmu_hw_events *hw_events,
1230               struct perf_event *event)
1231 {
1232     if (is_software_event(event))
1233         return 1;
1234 
1235     /*
1236      * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
1237      * core perf code won't check that the pmu->ctx == leader->ctx
1238      * until after pmu->event_init(event).
1239      */
1240     if (event->pmu != cci_pmu)
1241         return 0;
1242 
1243     if (event->state < PERF_EVENT_STATE_OFF)
1244         return 1;
1245 
1246     if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
1247         return 1;
1248 
1249     return pmu_get_event_idx(hw_events, event) >= 0;
1250 }
1251 
1252 static int validate_group(struct perf_event *event)
1253 {
1254     struct perf_event *sibling, *leader = event->group_leader;
1255     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1256     unsigned long mask[BITS_TO_LONGS(HW_CNTRS_MAX)];
1257     struct cci_pmu_hw_events fake_pmu = {
1258         /*
1259          * Initialise the fake PMU. We only need to populate the
1260          * used_mask for the purposes of validation.
1261          */
1262         .used_mask = mask,
1263     };
1264     bitmap_zero(mask, cci_pmu->num_cntrs);
1265 
1266     if (!validate_event(event->pmu, &fake_pmu, leader))
1267         return -EINVAL;
1268 
1269     for_each_sibling_event(sibling, leader) {
1270         if (!validate_event(event->pmu, &fake_pmu, sibling))
1271             return -EINVAL;
1272     }
1273 
1274     if (!validate_event(event->pmu, &fake_pmu, event))
1275         return -EINVAL;
1276 
1277     return 0;
1278 }
1279 
1280 static int __hw_perf_event_init(struct perf_event *event)
1281 {
1282     struct hw_perf_event *hwc = &event->hw;
1283     int mapping;
1284 
1285     mapping = pmu_map_event(event);
1286 
1287     if (mapping < 0) {
1288         pr_debug("event %x:%llx not supported\n", event->attr.type,
1289              event->attr.config);
1290         return mapping;
1291     }
1292 
1293     /*
1294      * We don't assign an index until we actually place the event onto
1295      * hardware. Use -1 to signify that we haven't decided where to put it
1296      * yet.
1297      */
1298     hwc->idx        = -1;
1299     hwc->config_base    = 0;
1300     hwc->config     = 0;
1301     hwc->event_base     = 0;
1302 
1303     /*
1304      * Store the event encoding into the config_base field.
1305      */
1306     hwc->config_base        |= (unsigned long)mapping;
1307 
1308     if (event->group_leader != event) {
1309         if (validate_group(event) != 0)
1310             return -EINVAL;
1311     }
1312 
1313     return 0;
1314 }
1315 
1316 static int cci_pmu_event_init(struct perf_event *event)
1317 {
1318     struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1319     atomic_t *active_events = &cci_pmu->active_events;
1320     int err = 0;
1321 
1322     if (event->attr.type != event->pmu->type)
1323         return -ENOENT;
1324 
1325     /* Shared by all CPUs, no meaningful state to sample */
1326     if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
1327         return -EOPNOTSUPP;
1328 
1329     /*
1330      * Following the example set by other "uncore" PMUs, we accept any CPU
1331      * and rewrite its affinity dynamically rather than having perf core
1332      * handle cpu == -1 and pid == -1 for this case.
1333      *
1334      * The perf core will pin online CPUs for the duration of this call and
1335      * the event being installed into its context, so the PMU's CPU can't
1336      * change under our feet.
1337      */
1338     if (event->cpu < 0)
1339         return -EINVAL;
1340     event->cpu = cci_pmu->cpu;
1341 
1342     event->destroy = hw_perf_event_destroy;
1343     if (!atomic_inc_not_zero(active_events)) {
1344         mutex_lock(&cci_pmu->reserve_mutex);
1345         if (atomic_read(active_events) == 0)
1346             err = cci_pmu_get_hw(cci_pmu);
1347         if (!err)
1348             atomic_inc(active_events);
1349         mutex_unlock(&cci_pmu->reserve_mutex);
1350     }
1351     if (err)
1352         return err;
1353 
1354     err = __hw_perf_event_init(event);
1355     if (err)
1356         hw_perf_event_destroy(event);
1357 
1358     return err;
1359 }
1360 
1361 static ssize_t pmu_cpumask_attr_show(struct device *dev,
1362                      struct device_attribute *attr, char *buf)
1363 {
1364     struct pmu *pmu = dev_get_drvdata(dev);
1365     struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1366 
1367     return cpumap_print_to_pagebuf(true, buf, cpumask_of(cci_pmu->cpu));
1368 }
1369 
1370 static struct device_attribute pmu_cpumask_attr =
1371     __ATTR(cpumask, S_IRUGO, pmu_cpumask_attr_show, NULL);
1372 
1373 static struct attribute *pmu_attrs[] = {
1374     &pmu_cpumask_attr.attr,
1375     NULL,
1376 };
1377 
1378 static const struct attribute_group pmu_attr_group = {
1379     .attrs = pmu_attrs,
1380 };
1381 
1382 static struct attribute_group pmu_format_attr_group = {
1383     .name = "format",
1384     .attrs = NULL,      /* Filled in cci_pmu_init_attrs */
1385 };
1386 
1387 static struct attribute_group pmu_event_attr_group = {
1388     .name = "events",
1389     .attrs = NULL,      /* Filled in cci_pmu_init_attrs */
1390 };
1391 
1392 static const struct attribute_group *pmu_attr_groups[] = {
1393     &pmu_attr_group,
1394     &pmu_format_attr_group,
1395     &pmu_event_attr_group,
1396     NULL
1397 };
1398 
1399 static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev)
1400 {
1401     const struct cci_pmu_model *model = cci_pmu->model;
1402     char *name = model->name;
1403     u32 num_cntrs;
1404 
1405     if (WARN_ON(model->num_hw_cntrs > NUM_HW_CNTRS_MAX))
1406         return -EINVAL;
1407     if (WARN_ON(model->fixed_hw_cntrs > FIXED_HW_CNTRS_MAX))
1408         return -EINVAL;
1409 
1410     pmu_event_attr_group.attrs = model->event_attrs;
1411     pmu_format_attr_group.attrs = model->format_attrs;
1412 
1413     cci_pmu->pmu = (struct pmu) {
1414         .module     = THIS_MODULE,
1415         .name       = cci_pmu->model->name,
1416         .task_ctx_nr    = perf_invalid_context,
1417         .pmu_enable = cci_pmu_enable,
1418         .pmu_disable    = cci_pmu_disable,
1419         .event_init = cci_pmu_event_init,
1420         .add        = cci_pmu_add,
1421         .del        = cci_pmu_del,
1422         .start      = cci_pmu_start,
1423         .stop       = cci_pmu_stop,
1424         .read       = pmu_read,
1425         .attr_groups    = pmu_attr_groups,
1426         .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
1427     };
1428 
1429     cci_pmu->plat_device = pdev;
1430     num_cntrs = pmu_get_max_counters(cci_pmu);
1431     if (num_cntrs > cci_pmu->model->num_hw_cntrs) {
1432         dev_warn(&pdev->dev,
1433             "PMU implements more counters(%d) than supported by"
1434             " the model(%d), truncated.",
1435             num_cntrs, cci_pmu->model->num_hw_cntrs);
1436         num_cntrs = cci_pmu->model->num_hw_cntrs;
1437     }
1438     cci_pmu->num_cntrs = num_cntrs + cci_pmu->model->fixed_hw_cntrs;
1439 
1440     return perf_pmu_register(&cci_pmu->pmu, name, -1);
1441 }
1442 
1443 static int cci_pmu_offline_cpu(unsigned int cpu)
1444 {
1445     int target;
1446 
1447     if (!g_cci_pmu || cpu != g_cci_pmu->cpu)
1448         return 0;
1449 
1450     target = cpumask_any_but(cpu_online_mask, cpu);
1451     if (target >= nr_cpu_ids)
1452         return 0;
1453 
1454     perf_pmu_migrate_context(&g_cci_pmu->pmu, cpu, target);
1455     g_cci_pmu->cpu = target;
1456     return 0;
1457 }
1458 
1459 static __maybe_unused struct cci_pmu_model cci_pmu_models[] = {
1460 #ifdef CONFIG_ARM_CCI400_PMU
1461     [CCI400_R0] = {
1462         .name = "CCI_400",
1463         .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_4XX, /* Cycle counter */
1464         .num_hw_cntrs = NUM_HW_CNTRS_CII_4XX,
1465         .cntr_size = SZ_4K,
1466         .format_attrs = cci400_pmu_format_attrs,
1467         .event_attrs = cci400_r0_pmu_event_attrs,
1468         .event_ranges = {
1469             [CCI_IF_SLAVE] = {
1470                 CCI400_R0_SLAVE_PORT_MIN_EV,
1471                 CCI400_R0_SLAVE_PORT_MAX_EV,
1472             },
1473             [CCI_IF_MASTER] = {
1474                 CCI400_R0_MASTER_PORT_MIN_EV,
1475                 CCI400_R0_MASTER_PORT_MAX_EV,
1476             },
1477         },
1478         .validate_hw_event = cci400_validate_hw_event,
1479         .get_event_idx = cci400_get_event_idx,
1480     },
1481     [CCI400_R1] = {
1482         .name = "CCI_400_r1",
1483         .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_4XX, /* Cycle counter */
1484         .num_hw_cntrs = NUM_HW_CNTRS_CII_4XX,
1485         .cntr_size = SZ_4K,
1486         .format_attrs = cci400_pmu_format_attrs,
1487         .event_attrs = cci400_r1_pmu_event_attrs,
1488         .event_ranges = {
1489             [CCI_IF_SLAVE] = {
1490                 CCI400_R1_SLAVE_PORT_MIN_EV,
1491                 CCI400_R1_SLAVE_PORT_MAX_EV,
1492             },
1493             [CCI_IF_MASTER] = {
1494                 CCI400_R1_MASTER_PORT_MIN_EV,
1495                 CCI400_R1_MASTER_PORT_MAX_EV,
1496             },
1497         },
1498         .validate_hw_event = cci400_validate_hw_event,
1499         .get_event_idx = cci400_get_event_idx,
1500     },
1501 #endif
1502 #ifdef CONFIG_ARM_CCI5xx_PMU
1503     [CCI500_R0] = {
1504         .name = "CCI_500",
1505         .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_5XX,
1506         .num_hw_cntrs = NUM_HW_CNTRS_CII_5XX,
1507         .cntr_size = SZ_64K,
1508         .format_attrs = cci5xx_pmu_format_attrs,
1509         .event_attrs = cci5xx_pmu_event_attrs,
1510         .event_ranges = {
1511             [CCI_IF_SLAVE] = {
1512                 CCI5xx_SLAVE_PORT_MIN_EV,
1513                 CCI5xx_SLAVE_PORT_MAX_EV,
1514             },
1515             [CCI_IF_MASTER] = {
1516                 CCI5xx_MASTER_PORT_MIN_EV,
1517                 CCI5xx_MASTER_PORT_MAX_EV,
1518             },
1519             [CCI_IF_GLOBAL] = {
1520                 CCI5xx_GLOBAL_PORT_MIN_EV,
1521                 CCI5xx_GLOBAL_PORT_MAX_EV,
1522             },
1523         },
1524         .validate_hw_event = cci500_validate_hw_event,
1525         .write_counters = cci5xx_pmu_write_counters,
1526     },
1527     [CCI550_R0] = {
1528         .name = "CCI_550",
1529         .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_5XX,
1530         .num_hw_cntrs = NUM_HW_CNTRS_CII_5XX,
1531         .cntr_size = SZ_64K,
1532         .format_attrs = cci5xx_pmu_format_attrs,
1533         .event_attrs = cci5xx_pmu_event_attrs,
1534         .event_ranges = {
1535             [CCI_IF_SLAVE] = {
1536                 CCI5xx_SLAVE_PORT_MIN_EV,
1537                 CCI5xx_SLAVE_PORT_MAX_EV,
1538             },
1539             [CCI_IF_MASTER] = {
1540                 CCI5xx_MASTER_PORT_MIN_EV,
1541                 CCI5xx_MASTER_PORT_MAX_EV,
1542             },
1543             [CCI_IF_GLOBAL] = {
1544                 CCI5xx_GLOBAL_PORT_MIN_EV,
1545                 CCI5xx_GLOBAL_PORT_MAX_EV,
1546             },
1547         },
1548         .validate_hw_event = cci550_validate_hw_event,
1549         .write_counters = cci5xx_pmu_write_counters,
1550     },
1551 #endif
1552 };
1553 
1554 static const struct of_device_id arm_cci_pmu_matches[] = {
1555 #ifdef CONFIG_ARM_CCI400_PMU
1556     {
1557         .compatible = "arm,cci-400-pmu",
1558         .data   = NULL,
1559     },
1560     {
1561         .compatible = "arm,cci-400-pmu,r0",
1562         .data   = &cci_pmu_models[CCI400_R0],
1563     },
1564     {
1565         .compatible = "arm,cci-400-pmu,r1",
1566         .data   = &cci_pmu_models[CCI400_R1],
1567     },
1568 #endif
1569 #ifdef CONFIG_ARM_CCI5xx_PMU
1570     {
1571         .compatible = "arm,cci-500-pmu,r0",
1572         .data = &cci_pmu_models[CCI500_R0],
1573     },
1574     {
1575         .compatible = "arm,cci-550-pmu,r0",
1576         .data = &cci_pmu_models[CCI550_R0],
1577     },
1578 #endif
1579     {},
1580 };
1581 MODULE_DEVICE_TABLE(of, arm_cci_pmu_matches);
1582 
1583 static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs)
1584 {
1585     int i;
1586 
1587     for (i = 0; i < nr_irqs; i++)
1588         if (irq == irqs[i])
1589             return true;
1590 
1591     return false;
1592 }
1593 
1594 static struct cci_pmu *cci_pmu_alloc(struct device *dev)
1595 {
1596     struct cci_pmu *cci_pmu;
1597     const struct cci_pmu_model *model;
1598 
1599     /*
1600      * All allocations are devm_* hence we don't have to free
1601      * them explicitly on an error, as it would end up in driver
1602      * detach.
1603      */
1604     cci_pmu = devm_kzalloc(dev, sizeof(*cci_pmu), GFP_KERNEL);
1605     if (!cci_pmu)
1606         return ERR_PTR(-ENOMEM);
1607 
1608     cci_pmu->ctrl_base = *(void __iomem **)dev->platform_data;
1609 
1610     model = of_device_get_match_data(dev);
1611     if (!model) {
1612         dev_warn(dev,
1613              "DEPRECATED compatible property, requires secure access to CCI registers");
1614         model = probe_cci_model(cci_pmu);
1615     }
1616     if (!model) {
1617         dev_warn(dev, "CCI PMU version not supported\n");
1618         return ERR_PTR(-ENODEV);
1619     }
1620 
1621     cci_pmu->model = model;
1622     cci_pmu->irqs = devm_kcalloc(dev, CCI_PMU_MAX_HW_CNTRS(model),
1623                     sizeof(*cci_pmu->irqs), GFP_KERNEL);
1624     if (!cci_pmu->irqs)
1625         return ERR_PTR(-ENOMEM);
1626     cci_pmu->hw_events.events = devm_kcalloc(dev,
1627                          CCI_PMU_MAX_HW_CNTRS(model),
1628                          sizeof(*cci_pmu->hw_events.events),
1629                          GFP_KERNEL);
1630     if (!cci_pmu->hw_events.events)
1631         return ERR_PTR(-ENOMEM);
1632     cci_pmu->hw_events.used_mask = devm_bitmap_zalloc(dev,
1633                               CCI_PMU_MAX_HW_CNTRS(model),
1634                               GFP_KERNEL);
1635     if (!cci_pmu->hw_events.used_mask)
1636         return ERR_PTR(-ENOMEM);
1637 
1638     return cci_pmu;
1639 }
1640 
1641 static int cci_pmu_probe(struct platform_device *pdev)
1642 {
1643     struct cci_pmu *cci_pmu;
1644     int i, ret, irq;
1645 
1646     cci_pmu = cci_pmu_alloc(&pdev->dev);
1647     if (IS_ERR(cci_pmu))
1648         return PTR_ERR(cci_pmu);
1649 
1650     cci_pmu->base = devm_platform_ioremap_resource(pdev, 0);
1651     if (IS_ERR(cci_pmu->base))
1652         return -ENOMEM;
1653 
1654     /*
1655      * CCI PMU has one overflow interrupt per counter; but some may be tied
1656      * together to a common interrupt.
1657      */
1658     cci_pmu->nr_irqs = 0;
1659     for (i = 0; i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model); i++) {
1660         irq = platform_get_irq(pdev, i);
1661         if (irq < 0)
1662             break;
1663 
1664         if (is_duplicate_irq(irq, cci_pmu->irqs, cci_pmu->nr_irqs))
1665             continue;
1666 
1667         cci_pmu->irqs[cci_pmu->nr_irqs++] = irq;
1668     }
1669 
1670     /*
1671      * Ensure that the device tree has as many interrupts as the number
1672      * of counters.
1673      */
1674     if (i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)) {
1675         dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n",
1676             i, CCI_PMU_MAX_HW_CNTRS(cci_pmu->model));
1677         return -EINVAL;
1678     }
1679 
1680     raw_spin_lock_init(&cci_pmu->hw_events.pmu_lock);
1681     mutex_init(&cci_pmu->reserve_mutex);
1682     atomic_set(&cci_pmu->active_events, 0);
1683 
1684     cci_pmu->cpu = raw_smp_processor_id();
1685     g_cci_pmu = cci_pmu;
1686     cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE,
1687                   "perf/arm/cci:online", NULL,
1688                   cci_pmu_offline_cpu);
1689 
1690     ret = cci_pmu_init(cci_pmu, pdev);
1691     if (ret)
1692         goto error_pmu_init;
1693 
1694     pr_info("ARM %s PMU driver probed", cci_pmu->model->name);
1695     return 0;
1696 
1697 error_pmu_init:
1698     cpuhp_remove_state(CPUHP_AP_PERF_ARM_CCI_ONLINE);
1699     g_cci_pmu = NULL;
1700     return ret;
1701 }
1702 
1703 static int cci_pmu_remove(struct platform_device *pdev)
1704 {
1705     if (!g_cci_pmu)
1706         return 0;
1707 
1708     cpuhp_remove_state(CPUHP_AP_PERF_ARM_CCI_ONLINE);
1709     perf_pmu_unregister(&g_cci_pmu->pmu);
1710     g_cci_pmu = NULL;
1711 
1712     return 0;
1713 }
1714 
1715 static struct platform_driver cci_pmu_driver = {
1716     .driver = {
1717            .name = DRIVER_NAME,
1718            .of_match_table = arm_cci_pmu_matches,
1719            .suppress_bind_attrs = true,
1720           },
1721     .probe = cci_pmu_probe,
1722     .remove = cci_pmu_remove,
1723 };
1724 
1725 module_platform_driver(cci_pmu_driver);
1726 MODULE_LICENSE("GPL v2");
1727 MODULE_DESCRIPTION("ARM CCI PMU support");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!