drivers/perf/marvell_cn10k_ddr_pmu.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /* Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver
0003  *
0004  * Copyright (C) 2021 Marvell.
0005  */
0006
0007 #include <linux/init.h>
0008 #include <linux/io.h>
0009 #include <linux/module.h>
0010 #include <linux/of.h>
0011 #include <linux/of_address.h>
0012 #include <linux/of_device.h>
0013 #include <linux/perf_event.h>
0014 #include <linux/hrtimer.h>
0015
0016 /* Performance Counters Operating Mode Control Registers */
0017 #define DDRC_PERF_CNT_OP_MODE_CTRL  0x8020
0018 #define OP_MODE_CTRL_VAL_MANNUAL    0x1
0019
0020 /* Performance Counters Start Operation Control Registers */
0021 #define DDRC_PERF_CNT_START_OP_CTRL 0x8028
0022 #define START_OP_CTRL_VAL_START     0x1ULL
0023 #define START_OP_CTRL_VAL_ACTIVE    0x2
0024
0025 /* Performance Counters End Operation Control Registers */
0026 #define DDRC_PERF_CNT_END_OP_CTRL   0x8030
0027 #define END_OP_CTRL_VAL_END     0x1ULL
0028
0029 /* Performance Counters End Status Registers */
0030 #define DDRC_PERF_CNT_END_STATUS        0x8038
0031 #define END_STATUS_VAL_END_TIMER_MODE_END   0x1
0032
0033 /* Performance Counters Configuration Registers */
0034 #define DDRC_PERF_CFG_BASE      0x8040
0035
0036 /* 8 Generic event counter + 2 fixed event counters */
0037 #define DDRC_PERF_NUM_GEN_COUNTERS  8
0038 #define DDRC_PERF_NUM_FIX_COUNTERS  2
0039 #define DDRC_PERF_READ_COUNTER_IDX  DDRC_PERF_NUM_GEN_COUNTERS
0040 #define DDRC_PERF_WRITE_COUNTER_IDX (DDRC_PERF_NUM_GEN_COUNTERS + 1)
0041 #define DDRC_PERF_NUM_COUNTERS      (DDRC_PERF_NUM_GEN_COUNTERS + \
0042                      DDRC_PERF_NUM_FIX_COUNTERS)
0043
0044 /* Generic event counter registers */
0045 #define DDRC_PERF_CFG(n)        (DDRC_PERF_CFG_BASE + 8 * (n))
0046 #define EVENT_ENABLE            BIT_ULL(63)
0047
0048 /* Two dedicated event counters for DDR reads and writes */
0049 #define EVENT_DDR_READS         101
0050 #define EVENT_DDR_WRITES        100
0051
0052 /*
0053  * programmable events IDs in programmable event counters.
0054  * DO NOT change these event-id numbers, they are used to
0055  * program event bitmap in h/w.
0056  */
0057 #define EVENT_OP_IS_ZQLATCH         55
0058 #define EVENT_OP_IS_ZQSTART         54
0059 #define EVENT_OP_IS_TCR_MRR         53
0060 #define EVENT_OP_IS_DQSOSC_MRR          52
0061 #define EVENT_OP_IS_DQSOSC_MPC          51
0062 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR  50
0063 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD  49
0064 #define EVENT_BSM_STARVATION            48
0065 #define EVENT_BSM_ALLOC             47
0066 #define EVENT_LPR_REQ_WITH_NOCREDIT     46
0067 #define EVENT_HPR_REQ_WITH_NOCREDIT     45
0068 #define EVENT_OP_IS_ZQCS            44
0069 #define EVENT_OP_IS_ZQCL            43
0070 #define EVENT_OP_IS_LOAD_MODE           42
0071 #define EVENT_OP_IS_SPEC_REF            41
0072 #define EVENT_OP_IS_CRIT_REF            40
0073 #define EVENT_OP_IS_REFRESH         39
0074 #define EVENT_OP_IS_ENTER_MPSM          35
0075 #define EVENT_OP_IS_ENTER_POWERDOWN     31
0076 #define EVENT_OP_IS_ENTER_SELFREF       27
0077 #define EVENT_WAW_HAZARD            26
0078 #define EVENT_RAW_HAZARD            25
0079 #define EVENT_WAR_HAZARD            24
0080 #define EVENT_WRITE_COMBINE         23
0081 #define EVENT_RDWR_TRANSITIONS          22
0082 #define EVENT_PRECHARGE_FOR_OTHER       21
0083 #define EVENT_PRECHARGE_FOR_RDWR        20
0084 #define EVENT_OP_IS_PRECHARGE           19
0085 #define EVENT_OP_IS_MWR             18
0086 #define EVENT_OP_IS_WR              17
0087 #define EVENT_OP_IS_RD              16
0088 #define EVENT_OP_IS_RD_ACTIVATE         15
0089 #define EVENT_OP_IS_RD_OR_WR            14
0090 #define EVENT_OP_IS_ACTIVATE            13
0091 #define EVENT_WR_XACT_WHEN_CRITICAL     12
0092 #define EVENT_LPR_XACT_WHEN_CRITICAL        11
0093 #define EVENT_HPR_XACT_WHEN_CRITICAL        10
0094 #define EVENT_DFI_RD_DATA_CYCLES        9
0095 #define EVENT_DFI_WR_DATA_CYCLES        8
0096 #define EVENT_ACT_BYPASS            7
0097 #define EVENT_READ_BYPASS           6
0098 #define EVENT_HIF_HI_PRI_RD         5
0099 #define EVENT_HIF_RMW               4
0100 #define EVENT_HIF_RD                3
0101 #define EVENT_HIF_WR                2
0102 #define EVENT_HIF_RD_OR_WR          1
0103
0104 /* Event counter value registers */
0105 #define DDRC_PERF_CNT_VALUE_BASE        0x8080
0106 #define DDRC_PERF_CNT_VALUE(n)  (DDRC_PERF_CNT_VALUE_BASE + 8 * (n))
0107
0108 /* Fixed event counter enable/disable register */
0109 #define DDRC_PERF_CNT_FREERUN_EN    0x80C0
0110 #define DDRC_PERF_FREERUN_WRITE_EN  0x1
0111 #define DDRC_PERF_FREERUN_READ_EN   0x2
0112
0113 /* Fixed event counter control register */
0114 #define DDRC_PERF_CNT_FREERUN_CTRL  0x80C8
0115 #define DDRC_FREERUN_WRITE_CNT_CLR  0x1
0116 #define DDRC_FREERUN_READ_CNT_CLR   0x2
0117
0118 /* Fixed event counter value register */
0119 #define DDRC_PERF_CNT_VALUE_WR_OP   0x80D0
0120 #define DDRC_PERF_CNT_VALUE_RD_OP   0x80D8
0121 #define DDRC_PERF_CNT_VALUE_OVERFLOW    BIT_ULL(48)
0122 #define DDRC_PERF_CNT_MAX_VALUE     GENMASK_ULL(48, 0)
0123
0124 struct cn10k_ddr_pmu {
0125     struct pmu pmu;
0126     void __iomem *base;
0127     unsigned int cpu;
0128     struct  device *dev;
0129     int active_events;
0130     struct perf_event *events[DDRC_PERF_NUM_COUNTERS];
0131     struct hrtimer hrtimer;
0132     struct hlist_node node;
0133 };
0134
0135 #define to_cn10k_ddr_pmu(p) container_of(p, struct cn10k_ddr_pmu, pmu)
0136
0137 static ssize_t cn10k_ddr_pmu_event_show(struct device *dev,
0138                     struct device_attribute *attr,
0139                     char *page)
0140 {
0141     struct perf_pmu_events_attr *pmu_attr;
0142
0143     pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
0144     return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
0145
0146 }
0147
0148 #define CN10K_DDR_PMU_EVENT_ATTR(_name, _id)                     \
0149     PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id)
0150
0151 static struct attribute *cn10k_ddr_perf_events_attrs[] = {
0152     CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
0153     CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
0154     CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
0155     CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
0156     CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
0157     CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
0158     CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
0159     CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES),
0160     CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES),
0161     CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
0162                     EVENT_HPR_XACT_WHEN_CRITICAL),
0163     CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
0164                     EVENT_LPR_XACT_WHEN_CRITICAL),
0165     CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
0166                     EVENT_WR_XACT_WHEN_CRITICAL),
0167     CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
0168     CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR),
0169     CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE),
0170     CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
0171     CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
0172     CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
0173     CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
0174     CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR),
0175     CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
0176                     EVENT_PRECHARGE_FOR_OTHER),
0177     CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
0178     CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
0179     CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
0180     CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
0181     CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
0182     CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
0183     CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN),
0184     CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),
0185     CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
0186     CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
0187     CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
0188     CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
0189     CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),
0190     CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),
0191     CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit,
0192                     EVENT_HPR_REQ_WITH_NOCREDIT),
0193     CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit,
0194                     EVENT_LPR_REQ_WITH_NOCREDIT),
0195     CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),
0196     CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),
0197     CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
0198                     EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),
0199     CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
0200                     EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),
0201     CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),
0202     CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),
0203     CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),
0204     CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),
0205     CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),
0206     /* Free run event counters */
0207     CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
0208     CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
0209     NULL
0210 };
0211
0212 static struct attribute_group cn10k_ddr_perf_events_attr_group = {
0213     .name = "events",
0214     .attrs = cn10k_ddr_perf_events_attrs,
0215 };
0216
0217 PMU_FORMAT_ATTR(event, "config:0-8");
0218
0219 static struct attribute *cn10k_ddr_perf_format_attrs[] = {
0220     &format_attr_event.attr,
0221     NULL,
0222 };
0223
0224 static struct attribute_group cn10k_ddr_perf_format_attr_group = {
0225     .name = "format",
0226     .attrs = cn10k_ddr_perf_format_attrs,
0227 };
0228
0229 static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev,
0230                        struct device_attribute *attr,
0231                        char *buf)
0232 {
0233     struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev);
0234
0235     return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
0236 }
0237
0238 static struct device_attribute cn10k_ddr_perf_cpumask_attr =
0239     __ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL);
0240
0241 static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = {
0242     &cn10k_ddr_perf_cpumask_attr.attr,
0243     NULL,
0244 };
0245
0246 static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = {
0247     .attrs = cn10k_ddr_perf_cpumask_attrs,
0248 };
0249
0250 static const struct attribute_group *cn10k_attr_groups[] = {
0251     &cn10k_ddr_perf_events_attr_group,
0252     &cn10k_ddr_perf_format_attr_group,
0253     &cn10k_ddr_perf_cpumask_attr_group,
0254     NULL,
0255 };
0256
0257 /* Default poll timeout is 100 sec, which is very sufficient for
0258  * 48 bit counter incremented max at 5.6 GT/s, which may take many
0259  * hours to overflow.
0260  */
0261 static unsigned long cn10k_ddr_pmu_poll_period_sec = 100;
0262 module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644);
0263
0264 static ktime_t cn10k_ddr_pmu_timer_period(void)
0265 {
0266     return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC);
0267 }
0268
0269 static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap)
0270 {
0271     switch (eventid) {
0272     case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD:
0273     case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH:
0274         *event_bitmap = (1ULL << (eventid - 1));
0275         break;
0276     case EVENT_OP_IS_ENTER_SELFREF:
0277     case EVENT_OP_IS_ENTER_POWERDOWN:
0278     case EVENT_OP_IS_ENTER_MPSM:
0279         *event_bitmap = (0xFULL << (eventid - 1));
0280         break;
0281     default:
0282         pr_err("%s Invalid eventid %d\n", __func__, eventid);
0283         return -EINVAL;
0284     }
0285
0286     return 0;
0287 }
0288
0289 static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu,
0290                     struct perf_event *event)
0291 {
0292     u8 config = event->attr.config;
0293     int i;
0294
0295     /* DDR read free-run counter index */
0296     if (config == EVENT_DDR_READS) {
0297         pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event;
0298         return DDRC_PERF_READ_COUNTER_IDX;
0299     }
0300
0301     /* DDR write free-run counter index */
0302     if (config == EVENT_DDR_WRITES) {
0303         pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event;
0304         return DDRC_PERF_WRITE_COUNTER_IDX;
0305     }
0306
0307     /* Allocate DDR generic counters */
0308     for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
0309         if (pmu->events[i] == NULL) {
0310             pmu->events[i] = event;
0311             return i;
0312         }
0313     }
0314
0315     return -ENOENT;
0316 }
0317
0318 static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter)
0319 {
0320     pmu->events[counter] = NULL;
0321 }
0322
0323 static int cn10k_ddr_perf_event_init(struct perf_event *event)
0324 {
0325     struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
0326     struct hw_perf_event *hwc = &event->hw;
0327
0328     if (event->attr.type != event->pmu->type)
0329         return -ENOENT;
0330
0331     if (is_sampling_event(event)) {
0332         dev_info(pmu->dev, "Sampling not supported!\n");
0333         return -EOPNOTSUPP;
0334     }
0335
0336     if (event->cpu < 0) {
0337         dev_warn(pmu->dev, "Can't provide per-task data!\n");
0338         return -EOPNOTSUPP;
0339     }
0340
0341     /*  We must NOT create groups containing mixed PMUs */
0342     if (event->group_leader->pmu != event->pmu &&
0343         !is_software_event(event->group_leader))
0344         return -EINVAL;
0345
0346     /* Set ownership of event to one CPU, same event can not be observed
0347      * on multiple cpus at same time.
0348      */
0349     event->cpu = pmu->cpu;
0350     hwc->idx = -1;
0351     return 0;
0352 }
0353
0354 static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu,
0355                       int counter, bool enable)
0356 {
0357     u32 reg;
0358     u64 val;
0359
0360     if (counter > DDRC_PERF_NUM_COUNTERS) {
0361         pr_err("Error: unsupported counter %d\n", counter);
0362         return;
0363     }
0364
0365     if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
0366         reg = DDRC_PERF_CFG(counter);
0367         val = readq_relaxed(pmu->base + reg);
0368
0369         if (enable)
0370             val |= EVENT_ENABLE;
0371         else
0372             val &= ~EVENT_ENABLE;
0373
0374         writeq_relaxed(val, pmu->base + reg);
0375     } else {
0376         val = readq_relaxed(pmu->base + DDRC_PERF_CNT_FREERUN_EN);
0377         if (enable) {
0378             if (counter == DDRC_PERF_READ_COUNTER_IDX)
0379                 val |= DDRC_PERF_FREERUN_READ_EN;
0380             else
0381                 val |= DDRC_PERF_FREERUN_WRITE_EN;
0382         } else {
0383             if (counter == DDRC_PERF_READ_COUNTER_IDX)
0384                 val &= ~DDRC_PERF_FREERUN_READ_EN;
0385             else
0386                 val &= ~DDRC_PERF_FREERUN_WRITE_EN;
0387         }
0388         writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_EN);
0389     }
0390 }
0391
0392 static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter)
0393 {
0394     u64 val;
0395
0396     if (counter == DDRC_PERF_READ_COUNTER_IDX)
0397         return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_RD_OP);
0398
0399     if (counter == DDRC_PERF_WRITE_COUNTER_IDX)
0400         return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_WR_OP);
0401
0402     val = readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE(counter));
0403     return val;
0404 }
0405
0406 static void cn10k_ddr_perf_event_update(struct perf_event *event)
0407 {
0408     struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
0409     struct hw_perf_event *hwc = &event->hw;
0410     u64 prev_count, new_count, mask;
0411
0412     do {
0413         prev_count = local64_read(&hwc->prev_count);
0414         new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
0415     } while (local64_xchg(&hwc->prev_count, new_count) != prev_count);
0416
0417     mask = DDRC_PERF_CNT_MAX_VALUE;
0418
0419     local64_add((new_count - prev_count) & mask, &event->count);
0420 }
0421
0422 static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags)
0423 {
0424     struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
0425     struct hw_perf_event *hwc = &event->hw;
0426     int counter = hwc->idx;
0427
0428     local64_set(&hwc->prev_count, 0);
0429
0430     cn10k_ddr_perf_counter_enable(pmu, counter, true);
0431
0432     hwc->state = 0;
0433 }
0434
0435 static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags)
0436 {
0437     struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
0438     struct hw_perf_event *hwc = &event->hw;
0439     u8 config = event->attr.config;
0440     int counter, ret;
0441     u32 reg_offset;
0442     u64 val;
0443
0444     counter = cn10k_ddr_perf_alloc_counter(pmu, event);
0445     if (counter < 0)
0446         return -EAGAIN;
0447
0448     pmu->active_events++;
0449     hwc->idx = counter;
0450
0451     if (pmu->active_events == 1)
0452         hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(),
0453                   HRTIMER_MODE_REL_PINNED);
0454
0455     if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
0456         /* Generic counters, configure event id */
0457         reg_offset = DDRC_PERF_CFG(counter);
0458         ret = ddr_perf_get_event_bitmap(config, &val);
0459         if (ret)
0460             return ret;
0461
0462         writeq_relaxed(val, pmu->base + reg_offset);
0463     } else {
0464         /* fixed event counter, clear counter value */
0465         if (counter == DDRC_PERF_READ_COUNTER_IDX)
0466             val = DDRC_FREERUN_READ_CNT_CLR;
0467         else
0468             val = DDRC_FREERUN_WRITE_CNT_CLR;
0469
0470         writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_CTRL);
0471     }
0472
0473     hwc->state |= PERF_HES_STOPPED;
0474
0475     if (flags & PERF_EF_START)
0476         cn10k_ddr_perf_event_start(event, flags);
0477
0478     return 0;
0479 }
0480
0481 static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags)
0482 {
0483     struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
0484     struct hw_perf_event *hwc = &event->hw;
0485     int counter = hwc->idx;
0486
0487     cn10k_ddr_perf_counter_enable(pmu, counter, false);
0488
0489     if (flags & PERF_EF_UPDATE)
0490         cn10k_ddr_perf_event_update(event);
0491
0492     hwc->state |= PERF_HES_STOPPED;
0493 }
0494
0495 static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags)
0496 {
0497     struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
0498     struct hw_perf_event *hwc = &event->hw;
0499     int counter = hwc->idx;
0500
0501     cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE);
0502
0503     cn10k_ddr_perf_free_counter(pmu, counter);
0504     pmu->active_events--;
0505     hwc->idx = -1;
0506
0507     /* Cancel timer when no events to capture */
0508     if (pmu->active_events == 0)
0509         hrtimer_cancel(&pmu->hrtimer);
0510 }
0511
0512 static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu)
0513 {
0514     struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
0515
0516     writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +
0517                DDRC_PERF_CNT_START_OP_CTRL);
0518 }
0519
0520 static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu)
0521 {
0522     struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
0523
0524     writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +
0525                DDRC_PERF_CNT_END_OP_CTRL);
0526 }
0527
0528 static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu)
0529 {
0530     struct hw_perf_event *hwc;
0531     int i;
0532
0533     for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
0534         if (pmu->events[i] == NULL)
0535             continue;
0536
0537         cn10k_ddr_perf_event_update(pmu->events[i]);
0538     }
0539
0540     /* Reset previous count as h/w counter are reset */
0541     for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
0542         if (pmu->events[i] == NULL)
0543             continue;
0544
0545         hwc = &pmu->events[i]->hw;
0546         local64_set(&hwc->prev_count, 0);
0547     }
0548 }
0549
0550 static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu)
0551 {
0552     struct perf_event *event;
0553     struct hw_perf_event *hwc;
0554     u64 prev_count, new_count;
0555     u64 value;
0556     int i;
0557
0558     event = pmu->events[DDRC_PERF_READ_COUNTER_IDX];
0559     if (event) {
0560         hwc = &event->hw;
0561         prev_count = local64_read(&hwc->prev_count);
0562         new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
0563
0564         /* Overflow condition is when new count less than
0565          * previous count
0566          */
0567         if (new_count < prev_count)
0568             cn10k_ddr_perf_event_update(event);
0569     }
0570
0571     event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX];
0572     if (event) {
0573         hwc = &event->hw;
0574         prev_count = local64_read(&hwc->prev_count);
0575         new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
0576
0577         /* Overflow condition is when new count less than
0578          * previous count
0579          */
0580         if (new_count < prev_count)
0581             cn10k_ddr_perf_event_update(event);
0582     }
0583
0584     for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
0585         if (pmu->events[i] == NULL)
0586             continue;
0587
0588         value = cn10k_ddr_perf_read_counter(pmu, i);
0589         if (value == DDRC_PERF_CNT_MAX_VALUE) {
0590             pr_info("Counter-(%d) reached max value\n", i);
0591             cn10k_ddr_perf_event_update_all(pmu);
0592             cn10k_ddr_perf_pmu_disable(&pmu->pmu);
0593             cn10k_ddr_perf_pmu_enable(&pmu->pmu);
0594         }
0595     }
0596
0597     return IRQ_HANDLED;
0598 }
0599
0600 static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer)
0601 {
0602     struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu,
0603                          hrtimer);
0604     unsigned long flags;
0605
0606     local_irq_save(flags);
0607     cn10k_ddr_pmu_overflow_handler(pmu);
0608     local_irq_restore(flags);
0609
0610     hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period());
0611     return HRTIMER_RESTART;
0612 }
0613
0614 static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
0615 {
0616     struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu,
0617                              node);
0618     unsigned int target;
0619
0620     if (cpu != pmu->cpu)
0621         return 0;
0622
0623     target = cpumask_any_but(cpu_online_mask, cpu);
0624     if (target >= nr_cpu_ids)
0625         return 0;
0626
0627     perf_pmu_migrate_context(&pmu->pmu, cpu, target);
0628     pmu->cpu = target;
0629     return 0;
0630 }
0631
0632 static int cn10k_ddr_perf_probe(struct platform_device *pdev)
0633 {
0634     struct cn10k_ddr_pmu *ddr_pmu;
0635     struct resource *res;
0636     void __iomem *base;
0637     char *name;
0638     int ret;
0639
0640     ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL);
0641     if (!ddr_pmu)
0642         return -ENOMEM;
0643
0644     ddr_pmu->dev = &pdev->dev;
0645     platform_set_drvdata(pdev, ddr_pmu);
0646
0647     base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
0648     if (IS_ERR(base))
0649         return PTR_ERR(base);
0650
0651     ddr_pmu->base = base;
0652
0653     /* Setup the PMU counter to work in manual mode */
0654     writeq_relaxed(OP_MODE_CTRL_VAL_MANNUAL, ddr_pmu->base +
0655                DDRC_PERF_CNT_OP_MODE_CTRL);
0656
0657     ddr_pmu->pmu = (struct pmu) {
0658         .module       = THIS_MODULE,
0659         .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
0660         .task_ctx_nr = perf_invalid_context,
0661         .attr_groups = cn10k_attr_groups,
0662         .event_init  = cn10k_ddr_perf_event_init,
0663         .add         = cn10k_ddr_perf_event_add,
0664         .del         = cn10k_ddr_perf_event_del,
0665         .start       = cn10k_ddr_perf_event_start,
0666         .stop        = cn10k_ddr_perf_event_stop,
0667         .read        = cn10k_ddr_perf_event_update,
0668         .pmu_enable  = cn10k_ddr_perf_pmu_enable,
0669         .pmu_disable = cn10k_ddr_perf_pmu_disable,
0670     };
0671
0672     /* Choose this cpu to collect perf data */
0673     ddr_pmu->cpu = raw_smp_processor_id();
0674
0675     name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx",
0676                   res->start);
0677     if (!name)
0678         return -ENOMEM;
0679
0680     hrtimer_init(&ddr_pmu->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
0681     ddr_pmu->hrtimer.function = cn10k_ddr_pmu_timer_handler;
0682
0683     cpuhp_state_add_instance_nocalls(
0684                 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
0685                 &ddr_pmu->node);
0686
0687     ret = perf_pmu_register(&ddr_pmu->pmu, name, -1);
0688     if (ret)
0689         goto error;
0690
0691     pr_info("CN10K DDR PMU Driver for ddrc@%llx\n", res->start);
0692     return 0;
0693 error:
0694     cpuhp_state_remove_instance_nocalls(
0695                 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
0696                 &ddr_pmu->node);
0697     return ret;
0698 }
0699
0700 static int cn10k_ddr_perf_remove(struct platform_device *pdev)
0701 {
0702     struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev);
0703
0704     cpuhp_state_remove_instance_nocalls(
0705                 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
0706                 &ddr_pmu->node);
0707
0708     perf_pmu_unregister(&ddr_pmu->pmu);
0709     return 0;
0710 }
0711
0712 #ifdef CONFIG_OF
0713 static const struct of_device_id cn10k_ddr_pmu_of_match[] = {
0714     { .compatible = "marvell,cn10k-ddr-pmu", },
0715     { },
0716 };
0717 MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match);
0718 #endif
0719
0720 static struct platform_driver cn10k_ddr_pmu_driver = {
0721     .driver = {
0722         .name   = "cn10k-ddr-pmu",
0723         .of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match),
0724         .suppress_bind_attrs = true,
0725     },
0726     .probe      = cn10k_ddr_perf_probe,
0727     .remove     = cn10k_ddr_perf_remove,
0728 };
0729
0730 static int __init cn10k_ddr_pmu_init(void)
0731 {
0732     int ret;
0733
0734     ret = cpuhp_setup_state_multi(
0735                 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
0736                 "perf/marvell/cn10k/ddr:online", NULL,
0737                 cn10k_ddr_pmu_offline_cpu);
0738     if (ret)
0739         return ret;
0740
0741     ret = platform_driver_register(&cn10k_ddr_pmu_driver);
0742     if (ret)
0743         cpuhp_remove_multi_state(
0744                 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
0745     return ret;
0746 }
0747
0748 static void __exit cn10k_ddr_pmu_exit(void)
0749 {
0750     platform_driver_unregister(&cn10k_ddr_pmu_driver);
0751     cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
0752 }
0753
0754 module_init(cn10k_ddr_pmu_init);
0755 module_exit(cn10k_ddr_pmu_exit);
0756
0757 MODULE_AUTHOR("Bharat Bhushan <bbhushan2@marvell.com>");
0758 MODULE_LICENSE("GPL v2");