drivers/perf/fsl_imx8_ddr_perf.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright 2017 NXP
0004  * Copyright 2016 Freescale Semiconductor, Inc.
0005  */
0006
0007 #include <linux/bitfield.h>
0008 #include <linux/init.h>
0009 #include <linux/interrupt.h>
0010 #include <linux/io.h>
0011 #include <linux/module.h>
0012 #include <linux/of.h>
0013 #include <linux/of_address.h>
0014 #include <linux/of_device.h>
0015 #include <linux/of_irq.h>
0016 #include <linux/perf_event.h>
0017 #include <linux/slab.h>
0018
0019 #define COUNTER_CNTL        0x0
0020 #define COUNTER_READ        0x20
0021
0022 #define COUNTER_DPCR1       0x30
0023
0024 #define CNTL_OVER       0x1
0025 #define CNTL_CLEAR      0x2
0026 #define CNTL_EN         0x4
0027 #define CNTL_EN_MASK        0xFFFFFFFB
0028 #define CNTL_CLEAR_MASK     0xFFFFFFFD
0029 #define CNTL_OVER_MASK      0xFFFFFFFE
0030
0031 #define CNTL_CSV_SHIFT      24
0032 #define CNTL_CSV_MASK       (0xFFU << CNTL_CSV_SHIFT)
0033
0034 #define EVENT_CYCLES_ID     0
0035 #define EVENT_CYCLES_COUNTER    0
0036 #define NUM_COUNTERS        4
0037
0038 #define AXI_MASKING_REVERT  0xffff0000  /* AXI_MASKING(MSB 16bits) + AXI_ID(LSB 16bits) */
0039
0040 #define to_ddr_pmu(p)       container_of(p, struct ddr_pmu, pmu)
0041
0042 #define DDR_PERF_DEV_NAME   "imx8_ddr"
0043 #define DDR_CPUHP_CB_NAME   DDR_PERF_DEV_NAME "_perf_pmu"
0044
0045 static DEFINE_IDA(ddr_ida);
0046
0047 /* DDR Perf hardware feature */
0048 #define DDR_CAP_AXI_ID_FILTER           0x1     /* support AXI ID filter */
0049 #define DDR_CAP_AXI_ID_FILTER_ENHANCED      0x3     /* support enhanced AXI ID filter */
0050
0051 struct fsl_ddr_devtype_data {
0052     unsigned int quirks;    /* quirks needed for different DDR Perf core */
0053     const char *identifier; /* system PMU identifier for userspace */
0054 };
0055
0056 static const struct fsl_ddr_devtype_data imx8_devtype_data;
0057
0058 static const struct fsl_ddr_devtype_data imx8m_devtype_data = {
0059     .quirks = DDR_CAP_AXI_ID_FILTER,
0060 };
0061
0062 static const struct fsl_ddr_devtype_data imx8mq_devtype_data = {
0063     .quirks = DDR_CAP_AXI_ID_FILTER,
0064     .identifier = "i.MX8MQ",
0065 };
0066
0067 static const struct fsl_ddr_devtype_data imx8mm_devtype_data = {
0068     .quirks = DDR_CAP_AXI_ID_FILTER,
0069     .identifier = "i.MX8MM",
0070 };
0071
0072 static const struct fsl_ddr_devtype_data imx8mn_devtype_data = {
0073     .quirks = DDR_CAP_AXI_ID_FILTER,
0074     .identifier = "i.MX8MN",
0075 };
0076
0077 static const struct fsl_ddr_devtype_data imx8mp_devtype_data = {
0078     .quirks = DDR_CAP_AXI_ID_FILTER_ENHANCED,
0079     .identifier = "i.MX8MP",
0080 };
0081
0082 static const struct of_device_id imx_ddr_pmu_dt_ids[] = {
0083     { .compatible = "fsl,imx8-ddr-pmu", .data = &imx8_devtype_data},
0084     { .compatible = "fsl,imx8m-ddr-pmu", .data = &imx8m_devtype_data},
0085     { .compatible = "fsl,imx8mq-ddr-pmu", .data = &imx8mq_devtype_data},
0086     { .compatible = "fsl,imx8mm-ddr-pmu", .data = &imx8mm_devtype_data},
0087     { .compatible = "fsl,imx8mn-ddr-pmu", .data = &imx8mn_devtype_data},
0088     { .compatible = "fsl,imx8mp-ddr-pmu", .data = &imx8mp_devtype_data},
0089     { /* sentinel */ }
0090 };
0091 MODULE_DEVICE_TABLE(of, imx_ddr_pmu_dt_ids);
0092
0093 struct ddr_pmu {
0094     struct pmu pmu;
0095     void __iomem *base;
0096     unsigned int cpu;
0097     struct  hlist_node node;
0098     struct  device *dev;
0099     struct perf_event *events[NUM_COUNTERS];
0100     int active_events;
0101     enum cpuhp_state cpuhp_state;
0102     const struct fsl_ddr_devtype_data *devtype_data;
0103     int irq;
0104     int id;
0105 };
0106
0107 static ssize_t ddr_perf_identifier_show(struct device *dev,
0108                     struct device_attribute *attr,
0109                     char *page)
0110 {
0111     struct ddr_pmu *pmu = dev_get_drvdata(dev);
0112
0113     return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);
0114 }
0115
0116 static umode_t ddr_perf_identifier_attr_visible(struct kobject *kobj,
0117                         struct attribute *attr,
0118                         int n)
0119 {
0120     struct device *dev = kobj_to_dev(kobj);
0121     struct ddr_pmu *pmu = dev_get_drvdata(dev);
0122
0123     if (!pmu->devtype_data->identifier)
0124         return 0;
0125     return attr->mode;
0126 };
0127
0128 static struct device_attribute ddr_perf_identifier_attr =
0129     __ATTR(identifier, 0444, ddr_perf_identifier_show, NULL);
0130
0131 static struct attribute *ddr_perf_identifier_attrs[] = {
0132     &ddr_perf_identifier_attr.attr,
0133     NULL,
0134 };
0135
0136 static const struct attribute_group ddr_perf_identifier_attr_group = {
0137     .attrs = ddr_perf_identifier_attrs,
0138     .is_visible = ddr_perf_identifier_attr_visible,
0139 };
0140
0141 enum ddr_perf_filter_capabilities {
0142     PERF_CAP_AXI_ID_FILTER = 0,
0143     PERF_CAP_AXI_ID_FILTER_ENHANCED,
0144     PERF_CAP_AXI_ID_FEAT_MAX,
0145 };
0146
0147 static u32 ddr_perf_filter_cap_get(struct ddr_pmu *pmu, int cap)
0148 {
0149     u32 quirks = pmu->devtype_data->quirks;
0150
0151     switch (cap) {
0152     case PERF_CAP_AXI_ID_FILTER:
0153         return !!(quirks & DDR_CAP_AXI_ID_FILTER);
0154     case PERF_CAP_AXI_ID_FILTER_ENHANCED:
0155         quirks &= DDR_CAP_AXI_ID_FILTER_ENHANCED;
0156         return quirks == DDR_CAP_AXI_ID_FILTER_ENHANCED;
0157     default:
0158         WARN(1, "unknown filter cap %d\n", cap);
0159     }
0160
0161     return 0;
0162 }
0163
0164 static ssize_t ddr_perf_filter_cap_show(struct device *dev,
0165                     struct device_attribute *attr,
0166                     char *buf)
0167 {
0168     struct ddr_pmu *pmu = dev_get_drvdata(dev);
0169     struct dev_ext_attribute *ea =
0170         container_of(attr, struct dev_ext_attribute, attr);
0171     int cap = (long)ea->var;
0172
0173     return sysfs_emit(buf, "%u\n", ddr_perf_filter_cap_get(pmu, cap));
0174 }
0175
0176 #define PERF_EXT_ATTR_ENTRY(_name, _func, _var)             \
0177     (&((struct dev_ext_attribute) {                 \
0178         __ATTR(_name, 0444, _func, NULL), (void *)_var      \
0179     }).attr.attr)
0180
0181 #define PERF_FILTER_EXT_ATTR_ENTRY(_name, _var)             \
0182     PERF_EXT_ATTR_ENTRY(_name, ddr_perf_filter_cap_show, _var)
0183
0184 static struct attribute *ddr_perf_filter_cap_attr[] = {
0185     PERF_FILTER_EXT_ATTR_ENTRY(filter, PERF_CAP_AXI_ID_FILTER),
0186     PERF_FILTER_EXT_ATTR_ENTRY(enhanced_filter, PERF_CAP_AXI_ID_FILTER_ENHANCED),
0187     NULL,
0188 };
0189
0190 static const struct attribute_group ddr_perf_filter_cap_attr_group = {
0191     .name = "caps",
0192     .attrs = ddr_perf_filter_cap_attr,
0193 };
0194
0195 static ssize_t ddr_perf_cpumask_show(struct device *dev,
0196                 struct device_attribute *attr, char *buf)
0197 {
0198     struct ddr_pmu *pmu = dev_get_drvdata(dev);
0199
0200     return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
0201 }
0202
0203 static struct device_attribute ddr_perf_cpumask_attr =
0204     __ATTR(cpumask, 0444, ddr_perf_cpumask_show, NULL);
0205
0206 static struct attribute *ddr_perf_cpumask_attrs[] = {
0207     &ddr_perf_cpumask_attr.attr,
0208     NULL,
0209 };
0210
0211 static const struct attribute_group ddr_perf_cpumask_attr_group = {
0212     .attrs = ddr_perf_cpumask_attrs,
0213 };
0214
0215 static ssize_t
0216 ddr_pmu_event_show(struct device *dev, struct device_attribute *attr,
0217            char *page)
0218 {
0219     struct perf_pmu_events_attr *pmu_attr;
0220
0221     pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
0222     return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
0223 }
0224
0225 #define IMX8_DDR_PMU_EVENT_ATTR(_name, _id)     \
0226     PMU_EVENT_ATTR_ID(_name, ddr_pmu_event_show, _id)
0227
0228 static struct attribute *ddr_perf_events_attrs[] = {
0229     IMX8_DDR_PMU_EVENT_ATTR(cycles, EVENT_CYCLES_ID),
0230     IMX8_DDR_PMU_EVENT_ATTR(selfresh, 0x01),
0231     IMX8_DDR_PMU_EVENT_ATTR(read-accesses, 0x04),
0232     IMX8_DDR_PMU_EVENT_ATTR(write-accesses, 0x05),
0233     IMX8_DDR_PMU_EVENT_ATTR(read-queue-depth, 0x08),
0234     IMX8_DDR_PMU_EVENT_ATTR(write-queue-depth, 0x09),
0235     IMX8_DDR_PMU_EVENT_ATTR(lp-read-credit-cnt, 0x10),
0236     IMX8_DDR_PMU_EVENT_ATTR(hp-read-credit-cnt, 0x11),
0237     IMX8_DDR_PMU_EVENT_ATTR(write-credit-cnt, 0x12),
0238     IMX8_DDR_PMU_EVENT_ATTR(read-command, 0x20),
0239     IMX8_DDR_PMU_EVENT_ATTR(write-command, 0x21),
0240     IMX8_DDR_PMU_EVENT_ATTR(read-modify-write-command, 0x22),
0241     IMX8_DDR_PMU_EVENT_ATTR(hp-read, 0x23),
0242     IMX8_DDR_PMU_EVENT_ATTR(hp-req-nocredit, 0x24),
0243     IMX8_DDR_PMU_EVENT_ATTR(hp-xact-credit, 0x25),
0244     IMX8_DDR_PMU_EVENT_ATTR(lp-req-nocredit, 0x26),
0245     IMX8_DDR_PMU_EVENT_ATTR(lp-xact-credit, 0x27),
0246     IMX8_DDR_PMU_EVENT_ATTR(wr-xact-credit, 0x29),
0247     IMX8_DDR_PMU_EVENT_ATTR(read-cycles, 0x2a),
0248     IMX8_DDR_PMU_EVENT_ATTR(write-cycles, 0x2b),
0249     IMX8_DDR_PMU_EVENT_ATTR(read-write-transition, 0x30),
0250     IMX8_DDR_PMU_EVENT_ATTR(precharge, 0x31),
0251     IMX8_DDR_PMU_EVENT_ATTR(activate, 0x32),
0252     IMX8_DDR_PMU_EVENT_ATTR(load-mode, 0x33),
0253     IMX8_DDR_PMU_EVENT_ATTR(perf-mwr, 0x34),
0254     IMX8_DDR_PMU_EVENT_ATTR(read, 0x35),
0255     IMX8_DDR_PMU_EVENT_ATTR(read-activate, 0x36),
0256     IMX8_DDR_PMU_EVENT_ATTR(refresh, 0x37),
0257     IMX8_DDR_PMU_EVENT_ATTR(write, 0x38),
0258     IMX8_DDR_PMU_EVENT_ATTR(raw-hazard, 0x39),
0259     IMX8_DDR_PMU_EVENT_ATTR(axid-read, 0x41),
0260     IMX8_DDR_PMU_EVENT_ATTR(axid-write, 0x42),
0261     NULL,
0262 };
0263
0264 static const struct attribute_group ddr_perf_events_attr_group = {
0265     .name = "events",
0266     .attrs = ddr_perf_events_attrs,
0267 };
0268
0269 PMU_FORMAT_ATTR(event, "config:0-7");
0270 PMU_FORMAT_ATTR(axi_id, "config1:0-15");
0271 PMU_FORMAT_ATTR(axi_mask, "config1:16-31");
0272
0273 static struct attribute *ddr_perf_format_attrs[] = {
0274     &format_attr_event.attr,
0275     &format_attr_axi_id.attr,
0276     &format_attr_axi_mask.attr,
0277     NULL,
0278 };
0279
0280 static const struct attribute_group ddr_perf_format_attr_group = {
0281     .name = "format",
0282     .attrs = ddr_perf_format_attrs,
0283 };
0284
0285 static const struct attribute_group *attr_groups[] = {
0286     &ddr_perf_events_attr_group,
0287     &ddr_perf_format_attr_group,
0288     &ddr_perf_cpumask_attr_group,
0289     &ddr_perf_filter_cap_attr_group,
0290     &ddr_perf_identifier_attr_group,
0291     NULL,
0292 };
0293
0294 static bool ddr_perf_is_filtered(struct perf_event *event)
0295 {
0296     return event->attr.config == 0x41 || event->attr.config == 0x42;
0297 }
0298
0299 static u32 ddr_perf_filter_val(struct perf_event *event)
0300 {
0301     return event->attr.config1;
0302 }
0303
0304 static bool ddr_perf_filters_compatible(struct perf_event *a,
0305                     struct perf_event *b)
0306 {
0307     if (!ddr_perf_is_filtered(a))
0308         return true;
0309     if (!ddr_perf_is_filtered(b))
0310         return true;
0311     return ddr_perf_filter_val(a) == ddr_perf_filter_val(b);
0312 }
0313
0314 static bool ddr_perf_is_enhanced_filtered(struct perf_event *event)
0315 {
0316     unsigned int filt;
0317     struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
0318
0319     filt = pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER_ENHANCED;
0320     return (filt == DDR_CAP_AXI_ID_FILTER_ENHANCED) &&
0321         ddr_perf_is_filtered(event);
0322 }
0323
0324 static u32 ddr_perf_alloc_counter(struct ddr_pmu *pmu, int event)
0325 {
0326     int i;
0327
0328     /*
0329      * Always map cycle event to counter 0
0330      * Cycles counter is dedicated for cycle event
0331      * can't used for the other events
0332      */
0333     if (event == EVENT_CYCLES_ID) {
0334         if (pmu->events[EVENT_CYCLES_COUNTER] == NULL)
0335             return EVENT_CYCLES_COUNTER;
0336         else
0337             return -ENOENT;
0338     }
0339
0340     for (i = 1; i < NUM_COUNTERS; i++) {
0341         if (pmu->events[i] == NULL)
0342             return i;
0343     }
0344
0345     return -ENOENT;
0346 }
0347
0348 static void ddr_perf_free_counter(struct ddr_pmu *pmu, int counter)
0349 {
0350     pmu->events[counter] = NULL;
0351 }
0352
0353 static u32 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter)
0354 {
0355     struct perf_event *event = pmu->events[counter];
0356     void __iomem *base = pmu->base;
0357
0358     /*
0359      * return bytes instead of bursts from ddr transaction for
0360      * axid-read and axid-write event if PMU core supports enhanced
0361      * filter.
0362      */
0363     base += ddr_perf_is_enhanced_filtered(event) ? COUNTER_DPCR1 :
0364                                COUNTER_READ;
0365     return readl_relaxed(base + counter * 4);
0366 }
0367
0368 static int ddr_perf_event_init(struct perf_event *event)
0369 {
0370     struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
0371     struct hw_perf_event *hwc = &event->hw;
0372     struct perf_event *sibling;
0373
0374     if (event->attr.type != event->pmu->type)
0375         return -ENOENT;
0376
0377     if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
0378         return -EOPNOTSUPP;
0379
0380     if (event->cpu < 0) {
0381         dev_warn(pmu->dev, "Can't provide per-task data!\n");
0382         return -EOPNOTSUPP;
0383     }
0384
0385     /*
0386      * We must NOT create groups containing mixed PMUs, although software
0387      * events are acceptable (for example to create a CCN group
0388      * periodically read when a hrtimer aka cpu-clock leader triggers).
0389      */
0390     if (event->group_leader->pmu != event->pmu &&
0391             !is_software_event(event->group_leader))
0392         return -EINVAL;
0393
0394     if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER) {
0395         if (!ddr_perf_filters_compatible(event, event->group_leader))
0396             return -EINVAL;
0397         for_each_sibling_event(sibling, event->group_leader) {
0398             if (!ddr_perf_filters_compatible(event, sibling))
0399                 return -EINVAL;
0400         }
0401     }
0402
0403     for_each_sibling_event(sibling, event->group_leader) {
0404         if (sibling->pmu != event->pmu &&
0405                 !is_software_event(sibling))
0406             return -EINVAL;
0407     }
0408
0409     event->cpu = pmu->cpu;
0410     hwc->idx = -1;
0411
0412     return 0;
0413 }
0414
0415 static void ddr_perf_counter_enable(struct ddr_pmu *pmu, int config,
0416                   int counter, bool enable)
0417 {
0418     u8 reg = counter * 4 + COUNTER_CNTL;
0419     int val;
0420
0421     if (enable) {
0422         /*
0423          * cycle counter is special which should firstly write 0 then
0424          * write 1 into CLEAR bit to clear it. Other counters only
0425          * need write 0 into CLEAR bit and it turns out to be 1 by
0426          * hardware. Below enable flow is harmless for all counters.
0427          */
0428         writel(0, pmu->base + reg);
0429         val = CNTL_EN | CNTL_CLEAR;
0430         val |= FIELD_PREP(CNTL_CSV_MASK, config);
0431         writel(val, pmu->base + reg);
0432     } else {
0433         /* Disable counter */
0434         val = readl_relaxed(pmu->base + reg) & CNTL_EN_MASK;
0435         writel(val, pmu->base + reg);
0436     }
0437 }
0438
0439 static bool ddr_perf_counter_overflow(struct ddr_pmu *pmu, int counter)
0440 {
0441     int val;
0442
0443     val = readl_relaxed(pmu->base + counter * 4 + COUNTER_CNTL);
0444
0445     return val & CNTL_OVER;
0446 }
0447
0448 static void ddr_perf_counter_clear(struct ddr_pmu *pmu, int counter)
0449 {
0450     u8 reg = counter * 4 + COUNTER_CNTL;
0451     int val;
0452
0453     val = readl_relaxed(pmu->base + reg);
0454     val &= ~CNTL_CLEAR;
0455     writel(val, pmu->base + reg);
0456
0457     val |= CNTL_CLEAR;
0458     writel(val, pmu->base + reg);
0459 }
0460
0461 static void ddr_perf_event_update(struct perf_event *event)
0462 {
0463     struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
0464     struct hw_perf_event *hwc = &event->hw;
0465     u64 new_raw_count;
0466     int counter = hwc->idx;
0467     int ret;
0468
0469     new_raw_count = ddr_perf_read_counter(pmu, counter);
0470     local64_add(new_raw_count, &event->count);
0471
0472     /*
0473      * For legacy SoCs: event counter continue counting when overflow,
0474      *                  no need to clear the counter.
0475      * For new SoCs: event counter stop counting when overflow, need
0476      *               clear counter to let it count again.
0477      */
0478     if (counter != EVENT_CYCLES_COUNTER) {
0479         ret = ddr_perf_counter_overflow(pmu, counter);
0480         if (ret)
0481             dev_warn_ratelimited(pmu->dev,  "events lost due to counter overflow (config 0x%llx)\n",
0482                          event->attr.config);
0483     }
0484
0485     /* clear counter every time for both cycle counter and event counter */
0486     ddr_perf_counter_clear(pmu, counter);
0487 }
0488
0489 static void ddr_perf_event_start(struct perf_event *event, int flags)
0490 {
0491     struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
0492     struct hw_perf_event *hwc = &event->hw;
0493     int counter = hwc->idx;
0494
0495     local64_set(&hwc->prev_count, 0);
0496
0497     ddr_perf_counter_enable(pmu, event->attr.config, counter, true);
0498
0499     hwc->state = 0;
0500 }
0501
0502 static int ddr_perf_event_add(struct perf_event *event, int flags)
0503 {
0504     struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
0505     struct hw_perf_event *hwc = &event->hw;
0506     int counter;
0507     int cfg = event->attr.config;
0508     int cfg1 = event->attr.config1;
0509
0510     if (pmu->devtype_data->quirks & DDR_CAP_AXI_ID_FILTER) {
0511         int i;
0512
0513         for (i = 1; i < NUM_COUNTERS; i++) {
0514             if (pmu->events[i] &&
0515                 !ddr_perf_filters_compatible(event, pmu->events[i]))
0516                 return -EINVAL;
0517         }
0518
0519         if (ddr_perf_is_filtered(event)) {
0520             /* revert axi id masking(axi_mask) value */
0521             cfg1 ^= AXI_MASKING_REVERT;
0522             writel(cfg1, pmu->base + COUNTER_DPCR1);
0523         }
0524     }
0525
0526     counter = ddr_perf_alloc_counter(pmu, cfg);
0527     if (counter < 0) {
0528         dev_dbg(pmu->dev, "There are not enough counters\n");
0529         return -EOPNOTSUPP;
0530     }
0531
0532     pmu->events[counter] = event;
0533     pmu->active_events++;
0534     hwc->idx = counter;
0535
0536     hwc->state |= PERF_HES_STOPPED;
0537
0538     if (flags & PERF_EF_START)
0539         ddr_perf_event_start(event, flags);
0540
0541     return 0;
0542 }
0543
0544 static void ddr_perf_event_stop(struct perf_event *event, int flags)
0545 {
0546     struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
0547     struct hw_perf_event *hwc = &event->hw;
0548     int counter = hwc->idx;
0549
0550     ddr_perf_counter_enable(pmu, event->attr.config, counter, false);
0551     ddr_perf_event_update(event);
0552
0553     hwc->state |= PERF_HES_STOPPED;
0554 }
0555
0556 static void ddr_perf_event_del(struct perf_event *event, int flags)
0557 {
0558     struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
0559     struct hw_perf_event *hwc = &event->hw;
0560     int counter = hwc->idx;
0561
0562     ddr_perf_event_stop(event, PERF_EF_UPDATE);
0563
0564     ddr_perf_free_counter(pmu, counter);
0565     pmu->active_events--;
0566     hwc->idx = -1;
0567 }
0568
0569 static void ddr_perf_pmu_enable(struct pmu *pmu)
0570 {
0571     struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
0572
0573     /* enable cycle counter if cycle is not active event list */
0574     if (ddr_pmu->events[EVENT_CYCLES_COUNTER] == NULL)
0575         ddr_perf_counter_enable(ddr_pmu,
0576                       EVENT_CYCLES_ID,
0577                       EVENT_CYCLES_COUNTER,
0578                       true);
0579 }
0580
0581 static void ddr_perf_pmu_disable(struct pmu *pmu)
0582 {
0583     struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
0584
0585     if (ddr_pmu->events[EVENT_CYCLES_COUNTER] == NULL)
0586         ddr_perf_counter_enable(ddr_pmu,
0587                       EVENT_CYCLES_ID,
0588                       EVENT_CYCLES_COUNTER,
0589                       false);
0590 }
0591
0592 static int ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base,
0593              struct device *dev)
0594 {
0595     *pmu = (struct ddr_pmu) {
0596         .pmu = (struct pmu) {
0597             .module       = THIS_MODULE,
0598             .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
0599             .task_ctx_nr = perf_invalid_context,
0600             .attr_groups = attr_groups,
0601             .event_init  = ddr_perf_event_init,
0602             .add         = ddr_perf_event_add,
0603             .del         = ddr_perf_event_del,
0604             .start       = ddr_perf_event_start,
0605             .stop        = ddr_perf_event_stop,
0606             .read        = ddr_perf_event_update,
0607             .pmu_enable  = ddr_perf_pmu_enable,
0608             .pmu_disable = ddr_perf_pmu_disable,
0609         },
0610         .base = base,
0611         .dev = dev,
0612     };
0613
0614     pmu->id = ida_alloc(&ddr_ida, GFP_KERNEL);
0615     return pmu->id;
0616 }
0617
0618 static irqreturn_t ddr_perf_irq_handler(int irq, void *p)
0619 {
0620     int i;
0621     struct ddr_pmu *pmu = (struct ddr_pmu *) p;
0622     struct perf_event *event;
0623
0624     /* all counter will stop if cycle counter disabled */
0625     ddr_perf_counter_enable(pmu,
0626                   EVENT_CYCLES_ID,
0627                   EVENT_CYCLES_COUNTER,
0628                   false);
0629     /*
0630      * When the cycle counter overflows, all counters are stopped,
0631      * and an IRQ is raised. If any other counter overflows, it
0632      * continues counting, and no IRQ is raised. But for new SoCs,
0633      * such as i.MX8MP, event counter would stop when overflow, so
0634      * we need use cycle counter to stop overflow of event counter.
0635      *
0636      * Cycles occur at least 4 times as often as other events, so we
0637      * can update all events on a cycle counter overflow and not
0638      * lose events.
0639      *
0640      */
0641     for (i = 0; i < NUM_COUNTERS; i++) {
0642
0643         if (!pmu->events[i])
0644             continue;
0645
0646         event = pmu->events[i];
0647
0648         ddr_perf_event_update(event);
0649     }
0650
0651     ddr_perf_counter_enable(pmu,
0652                   EVENT_CYCLES_ID,
0653                   EVENT_CYCLES_COUNTER,
0654                   true);
0655
0656     return IRQ_HANDLED;
0657 }
0658
0659 static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
0660 {
0661     struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
0662     int target;
0663
0664     if (cpu != pmu->cpu)
0665         return 0;
0666
0667     target = cpumask_any_but(cpu_online_mask, cpu);
0668     if (target >= nr_cpu_ids)
0669         return 0;
0670
0671     perf_pmu_migrate_context(&pmu->pmu, cpu, target);
0672     pmu->cpu = target;
0673
0674     WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));
0675
0676     return 0;
0677 }
0678
0679 static int ddr_perf_probe(struct platform_device *pdev)
0680 {
0681     struct ddr_pmu *pmu;
0682     struct device_node *np;
0683     void __iomem *base;
0684     char *name;
0685     int num;
0686     int ret;
0687     int irq;
0688
0689     base = devm_platform_ioremap_resource(pdev, 0);
0690     if (IS_ERR(base))
0691         return PTR_ERR(base);
0692
0693     np = pdev->dev.of_node;
0694
0695     pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);
0696     if (!pmu)
0697         return -ENOMEM;
0698
0699     num = ddr_perf_init(pmu, base, &pdev->dev);
0700
0701     platform_set_drvdata(pdev, pmu);
0702
0703     name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d",
0704                   num);
0705     if (!name) {
0706         ret = -ENOMEM;
0707         goto cpuhp_state_err;
0708     }
0709
0710     pmu->devtype_data = of_device_get_match_data(&pdev->dev);
0711
0712     pmu->cpu = raw_smp_processor_id();
0713     ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
0714                       DDR_CPUHP_CB_NAME,
0715                       NULL,
0716                       ddr_perf_offline_cpu);
0717
0718     if (ret < 0) {
0719         dev_err(&pdev->dev, "cpuhp_setup_state_multi failed\n");
0720         goto cpuhp_state_err;
0721     }
0722
0723     pmu->cpuhp_state = ret;
0724
0725     /* Register the pmu instance for cpu hotplug */
0726     ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
0727     if (ret) {
0728         dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
0729         goto cpuhp_instance_err;
0730     }
0731
0732     /* Request irq */
0733     irq = of_irq_get(np, 0);
0734     if (irq < 0) {
0735         dev_err(&pdev->dev, "Failed to get irq: %d", irq);
0736         ret = irq;
0737         goto ddr_perf_err;
0738     }
0739
0740     ret = devm_request_irq(&pdev->dev, irq,
0741                     ddr_perf_irq_handler,
0742                     IRQF_NOBALANCING | IRQF_NO_THREAD,
0743                     DDR_CPUHP_CB_NAME,
0744                     pmu);
0745     if (ret < 0) {
0746         dev_err(&pdev->dev, "Request irq failed: %d", ret);
0747         goto ddr_perf_err;
0748     }
0749
0750     pmu->irq = irq;
0751     ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));
0752     if (ret) {
0753         dev_err(pmu->dev, "Failed to set interrupt affinity!\n");
0754         goto ddr_perf_err;
0755     }
0756
0757     ret = perf_pmu_register(&pmu->pmu, name, -1);
0758     if (ret)
0759         goto ddr_perf_err;
0760
0761     return 0;
0762
0763 ddr_perf_err:
0764     cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
0765 cpuhp_instance_err:
0766     cpuhp_remove_multi_state(pmu->cpuhp_state);
0767 cpuhp_state_err:
0768     ida_free(&ddr_ida, pmu->id);
0769     dev_warn(&pdev->dev, "i.MX8 DDR Perf PMU failed (%d), disabled\n", ret);
0770     return ret;
0771 }
0772
0773 static int ddr_perf_remove(struct platform_device *pdev)
0774 {
0775     struct ddr_pmu *pmu = platform_get_drvdata(pdev);
0776
0777     cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
0778     cpuhp_remove_multi_state(pmu->cpuhp_state);
0779
0780     perf_pmu_unregister(&pmu->pmu);
0781
0782     ida_free(&ddr_ida, pmu->id);
0783     return 0;
0784 }
0785
0786 static struct platform_driver imx_ddr_pmu_driver = {
0787     .driver         = {
0788         .name   = "imx-ddr-pmu",
0789         .of_match_table = imx_ddr_pmu_dt_ids,
0790         .suppress_bind_attrs = true,
0791     },
0792     .probe          = ddr_perf_probe,
0793     .remove         = ddr_perf_remove,
0794 };
0795
0796 module_platform_driver(imx_ddr_pmu_driver);
0797 MODULE_LICENSE("GPL v2");