drivers/perf/arm_dmc620_pmu.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * ARM DMC-620 memory controller PMU driver
0004  *
0005  * Copyright (C) 2020 Ampere Computing LLC.
0006  */
0007
0008 #define DMC620_PMUNAME      "arm_dmc620"
0009 #define DMC620_DRVNAME      DMC620_PMUNAME "_pmu"
0010 #define pr_fmt(fmt)     DMC620_DRVNAME ": " fmt
0011
0012 #include <linux/acpi.h>
0013 #include <linux/bitfield.h>
0014 #include <linux/bitops.h>
0015 #include <linux/cpuhotplug.h>
0016 #include <linux/cpumask.h>
0017 #include <linux/device.h>
0018 #include <linux/errno.h>
0019 #include <linux/interrupt.h>
0020 #include <linux/irq.h>
0021 #include <linux/kernel.h>
0022 #include <linux/list.h>
0023 #include <linux/module.h>
0024 #include <linux/mutex.h>
0025 #include <linux/perf_event.h>
0026 #include <linux/platform_device.h>
0027 #include <linux/printk.h>
0028 #include <linux/rculist.h>
0029 #include <linux/refcount.h>
0030
0031 #define DMC620_PA_SHIFT                 12
0032 #define DMC620_CNT_INIT                 0x80000000
0033 #define DMC620_CNT_MAX_PERIOD               0xffffffff
0034 #define DMC620_PMU_CLKDIV2_MAX_COUNTERS         8
0035 #define DMC620_PMU_CLK_MAX_COUNTERS         2
0036 #define DMC620_PMU_MAX_COUNTERS             \
0037     (DMC620_PMU_CLKDIV2_MAX_COUNTERS + DMC620_PMU_CLK_MAX_COUNTERS)
0038
0039 /*
0040  * The PMU registers start at 0xA00 in the DMC-620 memory map, and these
0041  * offsets are relative to that base.
0042  *
0043  * Each counter has a group of control/value registers, and the
0044  * DMC620_PMU_COUNTERn offsets are within a counter group.
0045  *
0046  * The counter registers groups start at 0xA10.
0047  */
0048 #define DMC620_PMU_OVERFLOW_STATUS_CLKDIV2      0x8
0049 #define  DMC620_PMU_OVERFLOW_STATUS_CLKDIV2_MASK    \
0050         (DMC620_PMU_CLKDIV2_MAX_COUNTERS - 1)
0051 #define DMC620_PMU_OVERFLOW_STATUS_CLK          0xC
0052 #define  DMC620_PMU_OVERFLOW_STATUS_CLK_MASK        \
0053         (DMC620_PMU_CLK_MAX_COUNTERS - 1)
0054 #define DMC620_PMU_COUNTERS_BASE            0x10
0055 #define DMC620_PMU_COUNTERn_MASK_31_00          0x0
0056 #define DMC620_PMU_COUNTERn_MASK_63_32          0x4
0057 #define DMC620_PMU_COUNTERn_MATCH_31_00         0x8
0058 #define DMC620_PMU_COUNTERn_MATCH_63_32         0xC
0059 #define DMC620_PMU_COUNTERn_CONTROL         0x10
0060 #define  DMC620_PMU_COUNTERn_CONTROL_ENABLE     BIT(0)
0061 #define  DMC620_PMU_COUNTERn_CONTROL_INVERT     BIT(1)
0062 #define  DMC620_PMU_COUNTERn_CONTROL_EVENT_MUX      GENMASK(6, 2)
0063 #define  DMC620_PMU_COUNTERn_CONTROL_INCR_MUX       GENMASK(8, 7)
0064 #define DMC620_PMU_COUNTERn_VALUE           0x20
0065 /* Offset of the registers for a given counter, relative to 0xA00 */
0066 #define DMC620_PMU_COUNTERn_OFFSET(n) \
0067     (DMC620_PMU_COUNTERS_BASE + 0x28 * (n))
0068
0069 static LIST_HEAD(dmc620_pmu_irqs);
0070 static DEFINE_MUTEX(dmc620_pmu_irqs_lock);
0071
0072 struct dmc620_pmu_irq {
0073     struct hlist_node node;
0074     struct list_head pmus_node;
0075     struct list_head irqs_node;
0076     refcount_t refcount;
0077     unsigned int irq_num;
0078     unsigned int cpu;
0079 };
0080
0081 struct dmc620_pmu {
0082     struct pmu pmu;
0083
0084     void __iomem *base;
0085     struct dmc620_pmu_irq *irq;
0086     struct list_head pmus_node;
0087
0088     /*
0089      * We put all clkdiv2 and clk counters to a same array.
0090      * The first DMC620_PMU_CLKDIV2_MAX_COUNTERS bits belong to
0091      * clkdiv2 counters, the last DMC620_PMU_CLK_MAX_COUNTERS
0092      * belong to clk counters.
0093      */
0094     DECLARE_BITMAP(used_mask, DMC620_PMU_MAX_COUNTERS);
0095     struct perf_event *events[DMC620_PMU_MAX_COUNTERS];
0096 };
0097
0098 #define to_dmc620_pmu(p) (container_of(p, struct dmc620_pmu, pmu))
0099
0100 static int cpuhp_state_num;
0101
0102 struct dmc620_pmu_event_attr {
0103     struct device_attribute attr;
0104     u8 clkdiv2;
0105     u8 eventid;
0106 };
0107
0108 static ssize_t
0109 dmc620_pmu_event_show(struct device *dev,
0110                struct device_attribute *attr, char *page)
0111 {
0112     struct dmc620_pmu_event_attr *eattr;
0113
0114     eattr = container_of(attr, typeof(*eattr), attr);
0115
0116     return sysfs_emit(page, "event=0x%x,clkdiv2=0x%x\n", eattr->eventid, eattr->clkdiv2);
0117 }
0118
0119 #define DMC620_PMU_EVENT_ATTR(_name, _eventid, _clkdiv2)        \
0120     (&((struct dmc620_pmu_event_attr[]) {{              \
0121         .attr = __ATTR(_name, 0444, dmc620_pmu_event_show, NULL),   \
0122         .clkdiv2 = _clkdiv2,                        \
0123         .eventid = _eventid,                    \
0124     }})[0].attr.attr)
0125
0126 static struct attribute *dmc620_pmu_events_attrs[] = {
0127     /* clkdiv2 events list */
0128     DMC620_PMU_EVENT_ATTR(clkdiv2_cycle_count, 0x0, 1),
0129     DMC620_PMU_EVENT_ATTR(clkdiv2_allocate, 0x1, 1),
0130     DMC620_PMU_EVENT_ATTR(clkdiv2_queue_depth, 0x2, 1),
0131     DMC620_PMU_EVENT_ATTR(clkdiv2_waiting_for_wr_data, 0x3, 1),
0132     DMC620_PMU_EVENT_ATTR(clkdiv2_read_backlog, 0x4, 1),
0133     DMC620_PMU_EVENT_ATTR(clkdiv2_waiting_for_mi, 0x5, 1),
0134     DMC620_PMU_EVENT_ATTR(clkdiv2_hazard_resolution, 0x6, 1),
0135     DMC620_PMU_EVENT_ATTR(clkdiv2_enqueue, 0x7, 1),
0136     DMC620_PMU_EVENT_ATTR(clkdiv2_arbitrate, 0x8, 1),
0137     DMC620_PMU_EVENT_ATTR(clkdiv2_lrank_turnaround_activate, 0x9, 1),
0138     DMC620_PMU_EVENT_ATTR(clkdiv2_prank_turnaround_activate, 0xa, 1),
0139     DMC620_PMU_EVENT_ATTR(clkdiv2_read_depth, 0xb, 1),
0140     DMC620_PMU_EVENT_ATTR(clkdiv2_write_depth, 0xc, 1),
0141     DMC620_PMU_EVENT_ATTR(clkdiv2_highigh_qos_depth, 0xd, 1),
0142     DMC620_PMU_EVENT_ATTR(clkdiv2_high_qos_depth, 0xe, 1),
0143     DMC620_PMU_EVENT_ATTR(clkdiv2_medium_qos_depth, 0xf, 1),
0144     DMC620_PMU_EVENT_ATTR(clkdiv2_low_qos_depth, 0x10, 1),
0145     DMC620_PMU_EVENT_ATTR(clkdiv2_activate, 0x11, 1),
0146     DMC620_PMU_EVENT_ATTR(clkdiv2_rdwr, 0x12, 1),
0147     DMC620_PMU_EVENT_ATTR(clkdiv2_refresh, 0x13, 1),
0148     DMC620_PMU_EVENT_ATTR(clkdiv2_training_request, 0x14, 1),
0149     DMC620_PMU_EVENT_ATTR(clkdiv2_t_mac_tracker, 0x15, 1),
0150     DMC620_PMU_EVENT_ATTR(clkdiv2_bk_fsm_tracker, 0x16, 1),
0151     DMC620_PMU_EVENT_ATTR(clkdiv2_bk_open_tracker, 0x17, 1),
0152     DMC620_PMU_EVENT_ATTR(clkdiv2_ranks_in_pwr_down, 0x18, 1),
0153     DMC620_PMU_EVENT_ATTR(clkdiv2_ranks_in_sref, 0x19, 1),
0154
0155     /* clk events list */
0156     DMC620_PMU_EVENT_ATTR(clk_cycle_count, 0x0, 0),
0157     DMC620_PMU_EVENT_ATTR(clk_request, 0x1, 0),
0158     DMC620_PMU_EVENT_ATTR(clk_upload_stall, 0x2, 0),
0159     NULL,
0160 };
0161
0162 static const struct attribute_group dmc620_pmu_events_attr_group = {
0163     .name = "events",
0164     .attrs = dmc620_pmu_events_attrs,
0165 };
0166
0167 /* User ABI */
0168 #define ATTR_CFG_FLD_mask_CFG       config
0169 #define ATTR_CFG_FLD_mask_LO        0
0170 #define ATTR_CFG_FLD_mask_HI        44
0171 #define ATTR_CFG_FLD_match_CFG      config1
0172 #define ATTR_CFG_FLD_match_LO       0
0173 #define ATTR_CFG_FLD_match_HI       44
0174 #define ATTR_CFG_FLD_invert_CFG     config2
0175 #define ATTR_CFG_FLD_invert_LO      0
0176 #define ATTR_CFG_FLD_invert_HI      0
0177 #define ATTR_CFG_FLD_incr_CFG       config2
0178 #define ATTR_CFG_FLD_incr_LO        1
0179 #define ATTR_CFG_FLD_incr_HI        2
0180 #define ATTR_CFG_FLD_event_CFG      config2
0181 #define ATTR_CFG_FLD_event_LO       3
0182 #define ATTR_CFG_FLD_event_HI       8
0183 #define ATTR_CFG_FLD_clkdiv2_CFG    config2
0184 #define ATTR_CFG_FLD_clkdiv2_LO     9
0185 #define ATTR_CFG_FLD_clkdiv2_HI     9
0186
0187 #define __GEN_PMU_FORMAT_ATTR(cfg, lo, hi)          \
0188     (lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi
0189
0190 #define _GEN_PMU_FORMAT_ATTR(cfg, lo, hi)           \
0191     __GEN_PMU_FORMAT_ATTR(cfg, lo, hi)
0192
0193 #define GEN_PMU_FORMAT_ATTR(name)               \
0194     PMU_FORMAT_ATTR(name,                   \
0195     _GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG,     \
0196                  ATTR_CFG_FLD_##name##_LO,      \
0197                  ATTR_CFG_FLD_##name##_HI))
0198
0199 #define _ATTR_CFG_GET_FLD(attr, cfg, lo, hi)            \
0200     ((((attr)->cfg) >> lo) & GENMASK_ULL(hi - lo, 0))
0201
0202 #define ATTR_CFG_GET_FLD(attr, name)                \
0203     _ATTR_CFG_GET_FLD(attr,                 \
0204               ATTR_CFG_FLD_##name##_CFG,        \
0205               ATTR_CFG_FLD_##name##_LO,     \
0206               ATTR_CFG_FLD_##name##_HI)
0207
0208 GEN_PMU_FORMAT_ATTR(mask);
0209 GEN_PMU_FORMAT_ATTR(match);
0210 GEN_PMU_FORMAT_ATTR(invert);
0211 GEN_PMU_FORMAT_ATTR(incr);
0212 GEN_PMU_FORMAT_ATTR(event);
0213 GEN_PMU_FORMAT_ATTR(clkdiv2);
0214
0215 static struct attribute *dmc620_pmu_formats_attrs[] = {
0216     &format_attr_mask.attr,
0217     &format_attr_match.attr,
0218     &format_attr_invert.attr,
0219     &format_attr_incr.attr,
0220     &format_attr_event.attr,
0221     &format_attr_clkdiv2.attr,
0222     NULL,
0223 };
0224
0225 static const struct attribute_group dmc620_pmu_format_attr_group = {
0226     .name   = "format",
0227     .attrs  = dmc620_pmu_formats_attrs,
0228 };
0229
0230 static const struct attribute_group *dmc620_pmu_attr_groups[] = {
0231     &dmc620_pmu_events_attr_group,
0232     &dmc620_pmu_format_attr_group,
0233     NULL,
0234 };
0235
0236 static inline
0237 u32 dmc620_pmu_creg_read(struct dmc620_pmu *dmc620_pmu,
0238             unsigned int idx, unsigned int reg)
0239 {
0240     return readl(dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg);
0241 }
0242
0243 static inline
0244 void dmc620_pmu_creg_write(struct dmc620_pmu *dmc620_pmu,
0245             unsigned int idx, unsigned int reg, u32 val)
0246 {
0247     writel(val, dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg);
0248 }
0249
0250 static
0251 unsigned int dmc620_event_to_counter_control(struct perf_event *event)
0252 {
0253     struct perf_event_attr *attr = &event->attr;
0254     unsigned int reg = 0;
0255
0256     reg |= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_INVERT,
0257             ATTR_CFG_GET_FLD(attr, invert));
0258     reg |= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_EVENT_MUX,
0259             ATTR_CFG_GET_FLD(attr, event));
0260     reg |= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_INCR_MUX,
0261             ATTR_CFG_GET_FLD(attr, incr));
0262
0263     return reg;
0264 }
0265
0266 static int dmc620_get_event_idx(struct perf_event *event)
0267 {
0268     struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
0269     int idx, start_idx, end_idx;
0270
0271     if (ATTR_CFG_GET_FLD(&event->attr, clkdiv2)) {
0272         start_idx = 0;
0273         end_idx = DMC620_PMU_CLKDIV2_MAX_COUNTERS;
0274     } else {
0275         start_idx = DMC620_PMU_CLKDIV2_MAX_COUNTERS;
0276         end_idx = DMC620_PMU_MAX_COUNTERS;
0277     }
0278
0279     for (idx = start_idx; idx < end_idx; ++idx) {
0280         if (!test_and_set_bit(idx, dmc620_pmu->used_mask))
0281             return idx;
0282     }
0283
0284     /* The counters are all in use. */
0285     return -EAGAIN;
0286 }
0287
0288 static inline
0289 u64 dmc620_pmu_read_counter(struct perf_event *event)
0290 {
0291     struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
0292
0293     return dmc620_pmu_creg_read(dmc620_pmu,
0294                     event->hw.idx, DMC620_PMU_COUNTERn_VALUE);
0295 }
0296
0297 static void dmc620_pmu_event_update(struct perf_event *event)
0298 {
0299     struct hw_perf_event *hwc = &event->hw;
0300     u64 delta, prev_count, new_count;
0301
0302     do {
0303         /* We may also be called from the irq handler */
0304         prev_count = local64_read(&hwc->prev_count);
0305         new_count = dmc620_pmu_read_counter(event);
0306     } while (local64_cmpxchg(&hwc->prev_count,
0307             prev_count, new_count) != prev_count);
0308     delta = (new_count - prev_count) & DMC620_CNT_MAX_PERIOD;
0309     local64_add(delta, &event->count);
0310 }
0311
0312 static void dmc620_pmu_event_set_period(struct perf_event *event)
0313 {
0314     struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
0315
0316     local64_set(&event->hw.prev_count, DMC620_CNT_INIT);
0317     dmc620_pmu_creg_write(dmc620_pmu,
0318                   event->hw.idx, DMC620_PMU_COUNTERn_VALUE, DMC620_CNT_INIT);
0319 }
0320
0321 static void dmc620_pmu_enable_counter(struct perf_event *event)
0322 {
0323     struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
0324     u32 reg;
0325
0326     reg = dmc620_event_to_counter_control(event) | DMC620_PMU_COUNTERn_CONTROL_ENABLE;
0327     dmc620_pmu_creg_write(dmc620_pmu,
0328                   event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, reg);
0329 }
0330
0331 static void dmc620_pmu_disable_counter(struct perf_event *event)
0332 {
0333     struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
0334
0335     dmc620_pmu_creg_write(dmc620_pmu,
0336                   event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, 0);
0337 }
0338
0339 static irqreturn_t dmc620_pmu_handle_irq(int irq_num, void *data)
0340 {
0341     struct dmc620_pmu_irq *irq = data;
0342     struct dmc620_pmu *dmc620_pmu;
0343     irqreturn_t ret = IRQ_NONE;
0344
0345     rcu_read_lock();
0346     list_for_each_entry_rcu(dmc620_pmu, &irq->pmus_node, pmus_node) {
0347         unsigned long status;
0348         struct perf_event *event;
0349         unsigned int idx;
0350
0351         /*
0352          * HW doesn't provide a control to atomically disable all counters.
0353          * To prevent race condition (overflow happens while clearing status register),
0354          * disable all events before continuing
0355          */
0356         for (idx = 0; idx < DMC620_PMU_MAX_COUNTERS; idx++) {
0357             event = dmc620_pmu->events[idx];
0358             if (!event)
0359                 continue;
0360             dmc620_pmu_disable_counter(event);
0361         }
0362
0363         status = readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);
0364         status |= (readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK) <<
0365                 DMC620_PMU_CLKDIV2_MAX_COUNTERS);
0366         if (status) {
0367             for_each_set_bit(idx, &status,
0368                     DMC620_PMU_MAX_COUNTERS) {
0369                 event = dmc620_pmu->events[idx];
0370                 if (WARN_ON_ONCE(!event))
0371                     continue;
0372                 dmc620_pmu_event_update(event);
0373                 dmc620_pmu_event_set_period(event);
0374             }
0375
0376             if (status & DMC620_PMU_OVERFLOW_STATUS_CLKDIV2_MASK)
0377                 writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);
0378
0379             if ((status >> DMC620_PMU_CLKDIV2_MAX_COUNTERS) &
0380                 DMC620_PMU_OVERFLOW_STATUS_CLK_MASK)
0381                 writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK);
0382         }
0383
0384         for (idx = 0; idx < DMC620_PMU_MAX_COUNTERS; idx++) {
0385             event = dmc620_pmu->events[idx];
0386             if (!event)
0387                 continue;
0388             if (!(event->hw.state & PERF_HES_STOPPED))
0389                 dmc620_pmu_enable_counter(event);
0390         }
0391
0392         ret = IRQ_HANDLED;
0393     }
0394     rcu_read_unlock();
0395
0396     return ret;
0397 }
0398
0399 static struct dmc620_pmu_irq *__dmc620_pmu_get_irq(int irq_num)
0400 {
0401     struct dmc620_pmu_irq *irq;
0402     int ret;
0403
0404     list_for_each_entry(irq, &dmc620_pmu_irqs, irqs_node)
0405         if (irq->irq_num == irq_num && refcount_inc_not_zero(&irq->refcount))
0406             return irq;
0407
0408     irq = kzalloc(sizeof(*irq), GFP_KERNEL);
0409     if (!irq)
0410         return ERR_PTR(-ENOMEM);
0411
0412     INIT_LIST_HEAD(&irq->pmus_node);
0413
0414     /* Pick one CPU to be the preferred one to use */
0415     irq->cpu = raw_smp_processor_id();
0416     refcount_set(&irq->refcount, 1);
0417
0418     ret = request_irq(irq_num, dmc620_pmu_handle_irq,
0419               IRQF_NOBALANCING | IRQF_NO_THREAD,
0420               "dmc620-pmu", irq);
0421     if (ret)
0422         goto out_free_aff;
0423
0424     ret = irq_set_affinity(irq_num, cpumask_of(irq->cpu));
0425     if (ret)
0426         goto out_free_irq;
0427
0428     ret = cpuhp_state_add_instance_nocalls(cpuhp_state_num, &irq->node);
0429     if (ret)
0430         goto out_free_irq;
0431
0432     irq->irq_num = irq_num;
0433     list_add(&irq->irqs_node, &dmc620_pmu_irqs);
0434
0435     return irq;
0436
0437 out_free_irq:
0438     free_irq(irq_num, irq);
0439 out_free_aff:
0440     kfree(irq);
0441     return ERR_PTR(ret);
0442 }
0443
0444 static int dmc620_pmu_get_irq(struct dmc620_pmu *dmc620_pmu, int irq_num)
0445 {
0446     struct dmc620_pmu_irq *irq;
0447
0448     mutex_lock(&dmc620_pmu_irqs_lock);
0449     irq = __dmc620_pmu_get_irq(irq_num);
0450     mutex_unlock(&dmc620_pmu_irqs_lock);
0451
0452     if (IS_ERR(irq))
0453         return PTR_ERR(irq);
0454
0455     dmc620_pmu->irq = irq;
0456     mutex_lock(&dmc620_pmu_irqs_lock);
0457     list_add_rcu(&dmc620_pmu->pmus_node, &irq->pmus_node);
0458     mutex_unlock(&dmc620_pmu_irqs_lock);
0459
0460     return 0;
0461 }
0462
0463 static void dmc620_pmu_put_irq(struct dmc620_pmu *dmc620_pmu)
0464 {
0465     struct dmc620_pmu_irq *irq = dmc620_pmu->irq;
0466
0467     mutex_lock(&dmc620_pmu_irqs_lock);
0468     list_del_rcu(&dmc620_pmu->pmus_node);
0469
0470     if (!refcount_dec_and_test(&irq->refcount)) {
0471         mutex_unlock(&dmc620_pmu_irqs_lock);
0472         return;
0473     }
0474
0475     list_del(&irq->irqs_node);
0476     mutex_unlock(&dmc620_pmu_irqs_lock);
0477
0478     free_irq(irq->irq_num, irq);
0479     cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &irq->node);
0480     kfree(irq);
0481 }
0482
0483 static int dmc620_pmu_event_init(struct perf_event *event)
0484 {
0485     struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
0486     struct hw_perf_event *hwc = &event->hw;
0487     struct perf_event *sibling;
0488
0489     if (event->attr.type != event->pmu->type)
0490         return -ENOENT;
0491
0492     /*
0493      * DMC 620 PMUs are shared across all cpus and cannot
0494      * support task bound and sampling events.
0495      */
0496     if (is_sampling_event(event) ||
0497         event->attach_state & PERF_ATTACH_TASK) {
0498         dev_dbg(dmc620_pmu->pmu.dev,
0499             "Can't support per-task counters\n");
0500         return -EOPNOTSUPP;
0501     }
0502
0503     /*
0504      * Many perf core operations (eg. events rotation) operate on a
0505      * single CPU context. This is obvious for CPU PMUs, where one
0506      * expects the same sets of events being observed on all CPUs,
0507      * but can lead to issues for off-core PMUs, where each
0508      * event could be theoretically assigned to a different CPU. To
0509      * mitigate this, we enforce CPU assignment to one, selected
0510      * processor.
0511      */
0512     event->cpu = dmc620_pmu->irq->cpu;
0513     if (event->cpu < 0)
0514         return -EINVAL;
0515
0516     /*
0517      * We can't atomically disable all HW counters so only one event allowed,
0518      * although software events are acceptable.
0519      */
0520     if (event->group_leader != event &&
0521             !is_software_event(event->group_leader))
0522         return -EINVAL;
0523
0524     for_each_sibling_event(sibling, event->group_leader) {
0525         if (sibling != event &&
0526                 !is_software_event(sibling))
0527             return -EINVAL;
0528     }
0529
0530     hwc->idx = -1;
0531     return 0;
0532 }
0533
0534 static void dmc620_pmu_read(struct perf_event *event)
0535 {
0536     dmc620_pmu_event_update(event);
0537 }
0538
0539 static void dmc620_pmu_start(struct perf_event *event, int flags)
0540 {
0541     event->hw.state = 0;
0542     dmc620_pmu_event_set_period(event);
0543     dmc620_pmu_enable_counter(event);
0544 }
0545
0546 static void dmc620_pmu_stop(struct perf_event *event, int flags)
0547 {
0548     if (event->hw.state & PERF_HES_STOPPED)
0549         return;
0550
0551     dmc620_pmu_disable_counter(event);
0552     dmc620_pmu_event_update(event);
0553     event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
0554 }
0555
0556 static int dmc620_pmu_add(struct perf_event *event, int flags)
0557 {
0558     struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
0559     struct perf_event_attr *attr = &event->attr;
0560     struct hw_perf_event *hwc = &event->hw;
0561     int idx;
0562     u64 reg;
0563
0564     idx = dmc620_get_event_idx(event);
0565     if (idx < 0)
0566         return idx;
0567
0568     hwc->idx = idx;
0569     dmc620_pmu->events[idx] = event;
0570     hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
0571
0572     reg = ATTR_CFG_GET_FLD(attr, mask);
0573     dmc620_pmu_creg_write(dmc620_pmu,
0574                   idx, DMC620_PMU_COUNTERn_MASK_31_00, lower_32_bits(reg));
0575     dmc620_pmu_creg_write(dmc620_pmu,
0576                   idx, DMC620_PMU_COUNTERn_MASK_63_32, upper_32_bits(reg));
0577
0578     reg = ATTR_CFG_GET_FLD(attr, match);
0579     dmc620_pmu_creg_write(dmc620_pmu,
0580                   idx, DMC620_PMU_COUNTERn_MATCH_31_00, lower_32_bits(reg));
0581     dmc620_pmu_creg_write(dmc620_pmu,
0582                   idx, DMC620_PMU_COUNTERn_MATCH_63_32, upper_32_bits(reg));
0583
0584     if (flags & PERF_EF_START)
0585         dmc620_pmu_start(event, PERF_EF_RELOAD);
0586
0587     perf_event_update_userpage(event);
0588     return 0;
0589 }
0590
0591 static void dmc620_pmu_del(struct perf_event *event, int flags)
0592 {
0593     struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
0594     struct hw_perf_event *hwc = &event->hw;
0595     int idx = hwc->idx;
0596
0597     dmc620_pmu_stop(event, PERF_EF_UPDATE);
0598     dmc620_pmu->events[idx] = NULL;
0599     clear_bit(idx, dmc620_pmu->used_mask);
0600     perf_event_update_userpage(event);
0601 }
0602
0603 static int dmc620_pmu_cpu_teardown(unsigned int cpu,
0604                    struct hlist_node *node)
0605 {
0606     struct dmc620_pmu_irq *irq;
0607     struct dmc620_pmu *dmc620_pmu;
0608     unsigned int target;
0609
0610     irq = hlist_entry_safe(node, struct dmc620_pmu_irq, node);
0611     if (cpu != irq->cpu)
0612         return 0;
0613
0614     target = cpumask_any_but(cpu_online_mask, cpu);
0615     if (target >= nr_cpu_ids)
0616         return 0;
0617
0618     /* We're only reading, but this isn't the place to be involving RCU */
0619     mutex_lock(&dmc620_pmu_irqs_lock);
0620     list_for_each_entry(dmc620_pmu, &irq->pmus_node, pmus_node)
0621         perf_pmu_migrate_context(&dmc620_pmu->pmu, irq->cpu, target);
0622     mutex_unlock(&dmc620_pmu_irqs_lock);
0623
0624     WARN_ON(irq_set_affinity(irq->irq_num, cpumask_of(target)));
0625     irq->cpu = target;
0626
0627     return 0;
0628 }
0629
0630 static int dmc620_pmu_device_probe(struct platform_device *pdev)
0631 {
0632     struct dmc620_pmu *dmc620_pmu;
0633     struct resource *res;
0634     char *name;
0635     int irq_num;
0636     int i, ret;
0637
0638     dmc620_pmu = devm_kzalloc(&pdev->dev,
0639             sizeof(struct dmc620_pmu), GFP_KERNEL);
0640     if (!dmc620_pmu)
0641         return -ENOMEM;
0642
0643     platform_set_drvdata(pdev, dmc620_pmu);
0644
0645     dmc620_pmu->pmu = (struct pmu) {
0646         .module = THIS_MODULE,
0647         .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
0648         .task_ctx_nr    = perf_invalid_context,
0649         .event_init = dmc620_pmu_event_init,
0650         .add        = dmc620_pmu_add,
0651         .del        = dmc620_pmu_del,
0652         .start      = dmc620_pmu_start,
0653         .stop       = dmc620_pmu_stop,
0654         .read       = dmc620_pmu_read,
0655         .attr_groups    = dmc620_pmu_attr_groups,
0656     };
0657
0658     res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
0659     dmc620_pmu->base = devm_ioremap_resource(&pdev->dev, res);
0660     if (IS_ERR(dmc620_pmu->base))
0661         return PTR_ERR(dmc620_pmu->base);
0662
0663     /* Make sure device is reset before enabling interrupt */
0664     for (i = 0; i < DMC620_PMU_MAX_COUNTERS; i++)
0665         dmc620_pmu_creg_write(dmc620_pmu, i, DMC620_PMU_COUNTERn_CONTROL, 0);
0666     writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);
0667     writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK);
0668
0669     irq_num = platform_get_irq(pdev, 0);
0670     if (irq_num < 0)
0671         return irq_num;
0672
0673     ret = dmc620_pmu_get_irq(dmc620_pmu, irq_num);
0674     if (ret)
0675         return ret;
0676
0677     name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
0678                   "%s_%llx", DMC620_PMUNAME,
0679                   (u64)(res->start >> DMC620_PA_SHIFT));
0680     if (!name) {
0681         dev_err(&pdev->dev,
0682               "Create name failed, PMU @%pa\n", &res->start);
0683         ret = -ENOMEM;
0684         goto out_teardown_dev;
0685     }
0686
0687     ret = perf_pmu_register(&dmc620_pmu->pmu, name, -1);
0688     if (ret)
0689         goto out_teardown_dev;
0690
0691     return 0;
0692
0693 out_teardown_dev:
0694     dmc620_pmu_put_irq(dmc620_pmu);
0695     synchronize_rcu();
0696     return ret;
0697 }
0698
0699 static int dmc620_pmu_device_remove(struct platform_device *pdev)
0700 {
0701     struct dmc620_pmu *dmc620_pmu = platform_get_drvdata(pdev);
0702
0703     dmc620_pmu_put_irq(dmc620_pmu);
0704
0705     /* perf will synchronise RCU before devres can free dmc620_pmu */
0706     perf_pmu_unregister(&dmc620_pmu->pmu);
0707
0708     return 0;
0709 }
0710
0711 static const struct acpi_device_id dmc620_acpi_match[] = {
0712     { "ARMHD620", 0},
0713     {},
0714 };
0715 MODULE_DEVICE_TABLE(acpi, dmc620_acpi_match);
0716 static struct platform_driver dmc620_pmu_driver = {
0717     .driver = {
0718         .name       = DMC620_DRVNAME,
0719         .acpi_match_table = dmc620_acpi_match,
0720         .suppress_bind_attrs = true,
0721     },
0722     .probe  = dmc620_pmu_device_probe,
0723     .remove = dmc620_pmu_device_remove,
0724 };
0725
0726 static int __init dmc620_pmu_init(void)
0727 {
0728     cpuhp_state_num = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
0729                       DMC620_DRVNAME,
0730                       NULL,
0731                       dmc620_pmu_cpu_teardown);
0732     if (cpuhp_state_num < 0)
0733         return cpuhp_state_num;
0734
0735     return platform_driver_register(&dmc620_pmu_driver);
0736 }
0737
0738 static void __exit dmc620_pmu_exit(void)
0739 {
0740     platform_driver_unregister(&dmc620_pmu_driver);
0741     cpuhp_remove_multi_state(cpuhp_state_num);
0742 }
0743
0744 module_init(dmc620_pmu_init);
0745 module_exit(dmc620_pmu_exit);
0746
0747 MODULE_DESCRIPTION("Perf driver for the ARM DMC-620 memory controller");
0748 MODULE_AUTHOR("Tuan Phan <tuanphan@os.amperecomputing.com");
0749 MODULE_LICENSE("GPL v2");