Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * CAVIUM THUNDERX2 SoC PMU UNCORE
0004  * Copyright (C) 2018 Cavium Inc.
0005  * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
0006  */
0007 
0008 #include <linux/acpi.h>
0009 #include <linux/cpuhotplug.h>
0010 #include <linux/perf_event.h>
0011 #include <linux/platform_device.h>
0012 
0013 /* Each ThunderX2(TX2) Socket has a L3C and DMC UNCORE PMU device.
0014  * Each UNCORE PMU device consists of 4 independent programmable counters.
0015  * Counters are 32 bit and do not support overflow interrupt,
0016  * they need to be sampled before overflow(i.e, at every 2 seconds).
0017  */
0018 
0019 #define TX2_PMU_DMC_L3C_MAX_COUNTERS    4
0020 #define TX2_PMU_CCPI2_MAX_COUNTERS  8
0021 #define TX2_PMU_MAX_COUNTERS        TX2_PMU_CCPI2_MAX_COUNTERS
0022 
0023 
0024 #define TX2_PMU_DMC_CHANNELS        8
0025 #define TX2_PMU_L3_TILES        16
0026 
0027 #define TX2_PMU_HRTIMER_INTERVAL    (2 * NSEC_PER_SEC)
0028 #define GET_EVENTID(ev, mask)       ((ev->hw.config) & mask)
0029 #define GET_COUNTERID(ev, mask)     ((ev->hw.idx) & mask)
0030  /* 1 byte per counter(4 counters).
0031   * Event id is encoded in bits [5:1] of a byte,
0032   */
0033 #define DMC_EVENT_CFG(idx, val)     ((val) << (((idx) * 8) + 1))
0034 
0035 /* bits[3:0] to select counters, are indexed from 8 to 15. */
0036 #define CCPI2_COUNTER_OFFSET        8
0037 
0038 #define L3C_COUNTER_CTL         0xA8
0039 #define L3C_COUNTER_DATA        0xAC
0040 #define DMC_COUNTER_CTL         0x234
0041 #define DMC_COUNTER_DATA        0x240
0042 
0043 #define CCPI2_PERF_CTL          0x108
0044 #define CCPI2_COUNTER_CTL       0x10C
0045 #define CCPI2_COUNTER_SEL       0x12c
0046 #define CCPI2_COUNTER_DATA_L        0x130
0047 #define CCPI2_COUNTER_DATA_H        0x134
0048 
0049 /* L3C event IDs */
0050 #define L3_EVENT_READ_REQ       0xD
0051 #define L3_EVENT_WRITEBACK_REQ      0xE
0052 #define L3_EVENT_INV_N_WRITE_REQ    0xF
0053 #define L3_EVENT_INV_REQ        0x10
0054 #define L3_EVENT_EVICT_REQ      0x13
0055 #define L3_EVENT_INV_N_WRITE_HIT    0x14
0056 #define L3_EVENT_INV_HIT        0x15
0057 #define L3_EVENT_READ_HIT       0x17
0058 #define L3_EVENT_MAX            0x18
0059 
0060 /* DMC event IDs */
0061 #define DMC_EVENT_COUNT_CYCLES      0x1
0062 #define DMC_EVENT_WRITE_TXNS        0xB
0063 #define DMC_EVENT_DATA_TRANSFERS    0xD
0064 #define DMC_EVENT_READ_TXNS     0xF
0065 #define DMC_EVENT_MAX           0x10
0066 
0067 #define CCPI2_EVENT_REQ_PKT_SENT    0x3D
0068 #define CCPI2_EVENT_SNOOP_PKT_SENT  0x65
0069 #define CCPI2_EVENT_DATA_PKT_SENT   0x105
0070 #define CCPI2_EVENT_GIC_PKT_SENT    0x12D
0071 #define CCPI2_EVENT_MAX         0x200
0072 
0073 #define CCPI2_PERF_CTL_ENABLE       BIT(0)
0074 #define CCPI2_PERF_CTL_START        BIT(1)
0075 #define CCPI2_PERF_CTL_RESET        BIT(4)
0076 #define CCPI2_EVENT_LEVEL_RISING_EDGE   BIT(10)
0077 #define CCPI2_EVENT_TYPE_EDGE_SENSITIVE BIT(11)
0078 
0079 enum tx2_uncore_type {
0080     PMU_TYPE_L3C,
0081     PMU_TYPE_DMC,
0082     PMU_TYPE_CCPI2,
0083     PMU_TYPE_INVALID,
0084 };
0085 
0086 /*
0087  * Each socket has 3 uncore devices associated with a PMU. The DMC and
0088  * L3C have 4 32-bit counters and the CCPI2 has 8 64-bit counters.
0089  */
0090 struct tx2_uncore_pmu {
0091     struct hlist_node hpnode;
0092     struct list_head  entry;
0093     struct pmu pmu;
0094     char *name;
0095     int node;
0096     int cpu;
0097     u32 max_counters;
0098     u32 counters_mask;
0099     u32 prorate_factor;
0100     u32 max_events;
0101     u32 events_mask;
0102     u64 hrtimer_interval;
0103     void __iomem *base;
0104     DECLARE_BITMAP(active_counters, TX2_PMU_MAX_COUNTERS);
0105     struct perf_event *events[TX2_PMU_MAX_COUNTERS];
0106     struct device *dev;
0107     struct hrtimer hrtimer;
0108     const struct attribute_group **attr_groups;
0109     enum tx2_uncore_type type;
0110     enum hrtimer_restart (*hrtimer_callback)(struct hrtimer *cb);
0111     void (*init_cntr_base)(struct perf_event *event,
0112             struct tx2_uncore_pmu *tx2_pmu);
0113     void (*stop_event)(struct perf_event *event);
0114     void (*start_event)(struct perf_event *event, int flags);
0115 };
0116 
0117 static LIST_HEAD(tx2_pmus);
0118 
0119 static inline struct tx2_uncore_pmu *pmu_to_tx2_pmu(struct pmu *pmu)
0120 {
0121     return container_of(pmu, struct tx2_uncore_pmu, pmu);
0122 }
0123 
0124 #define TX2_PMU_FORMAT_ATTR(_var, _name, _format)           \
0125 static ssize_t                              \
0126 __tx2_pmu_##_var##_show(struct device *dev,             \
0127                    struct device_attribute *attr,       \
0128                    char *page)              \
0129 {                                   \
0130     BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);         \
0131     return sysfs_emit(page, _format "\n");              \
0132 }                                   \
0133                                     \
0134 static struct device_attribute format_attr_##_var =         \
0135     __ATTR(_name, 0444, __tx2_pmu_##_var##_show, NULL)
0136 
0137 TX2_PMU_FORMAT_ATTR(event, event, "config:0-4");
0138 TX2_PMU_FORMAT_ATTR(event_ccpi2, event, "config:0-9");
0139 
0140 static struct attribute *l3c_pmu_format_attrs[] = {
0141     &format_attr_event.attr,
0142     NULL,
0143 };
0144 
0145 static struct attribute *dmc_pmu_format_attrs[] = {
0146     &format_attr_event.attr,
0147     NULL,
0148 };
0149 
0150 static struct attribute *ccpi2_pmu_format_attrs[] = {
0151     &format_attr_event_ccpi2.attr,
0152     NULL,
0153 };
0154 
0155 static const struct attribute_group l3c_pmu_format_attr_group = {
0156     .name = "format",
0157     .attrs = l3c_pmu_format_attrs,
0158 };
0159 
0160 static const struct attribute_group dmc_pmu_format_attr_group = {
0161     .name = "format",
0162     .attrs = dmc_pmu_format_attrs,
0163 };
0164 
0165 static const struct attribute_group ccpi2_pmu_format_attr_group = {
0166     .name = "format",
0167     .attrs = ccpi2_pmu_format_attrs,
0168 };
0169 
0170 /*
0171  * sysfs event attributes
0172  */
0173 static ssize_t tx2_pmu_event_show(struct device *dev,
0174                     struct device_attribute *attr, char *buf)
0175 {
0176     struct dev_ext_attribute *eattr;
0177 
0178     eattr = container_of(attr, struct dev_ext_attribute, attr);
0179     return sysfs_emit(buf, "event=0x%lx\n", (unsigned long) eattr->var);
0180 }
0181 
0182 #define TX2_EVENT_ATTR(name, config) \
0183     PMU_EVENT_ATTR(name, tx2_pmu_event_attr_##name, \
0184             config, tx2_pmu_event_show)
0185 
0186 TX2_EVENT_ATTR(read_request, L3_EVENT_READ_REQ);
0187 TX2_EVENT_ATTR(writeback_request, L3_EVENT_WRITEBACK_REQ);
0188 TX2_EVENT_ATTR(inv_nwrite_request, L3_EVENT_INV_N_WRITE_REQ);
0189 TX2_EVENT_ATTR(inv_request, L3_EVENT_INV_REQ);
0190 TX2_EVENT_ATTR(evict_request, L3_EVENT_EVICT_REQ);
0191 TX2_EVENT_ATTR(inv_nwrite_hit, L3_EVENT_INV_N_WRITE_HIT);
0192 TX2_EVENT_ATTR(inv_hit, L3_EVENT_INV_HIT);
0193 TX2_EVENT_ATTR(read_hit, L3_EVENT_READ_HIT);
0194 
0195 static struct attribute *l3c_pmu_events_attrs[] = {
0196     &tx2_pmu_event_attr_read_request.attr.attr,
0197     &tx2_pmu_event_attr_writeback_request.attr.attr,
0198     &tx2_pmu_event_attr_inv_nwrite_request.attr.attr,
0199     &tx2_pmu_event_attr_inv_request.attr.attr,
0200     &tx2_pmu_event_attr_evict_request.attr.attr,
0201     &tx2_pmu_event_attr_inv_nwrite_hit.attr.attr,
0202     &tx2_pmu_event_attr_inv_hit.attr.attr,
0203     &tx2_pmu_event_attr_read_hit.attr.attr,
0204     NULL,
0205 };
0206 
0207 TX2_EVENT_ATTR(cnt_cycles, DMC_EVENT_COUNT_CYCLES);
0208 TX2_EVENT_ATTR(write_txns, DMC_EVENT_WRITE_TXNS);
0209 TX2_EVENT_ATTR(data_transfers, DMC_EVENT_DATA_TRANSFERS);
0210 TX2_EVENT_ATTR(read_txns, DMC_EVENT_READ_TXNS);
0211 
0212 static struct attribute *dmc_pmu_events_attrs[] = {
0213     &tx2_pmu_event_attr_cnt_cycles.attr.attr,
0214     &tx2_pmu_event_attr_write_txns.attr.attr,
0215     &tx2_pmu_event_attr_data_transfers.attr.attr,
0216     &tx2_pmu_event_attr_read_txns.attr.attr,
0217     NULL,
0218 };
0219 
0220 TX2_EVENT_ATTR(req_pktsent, CCPI2_EVENT_REQ_PKT_SENT);
0221 TX2_EVENT_ATTR(snoop_pktsent, CCPI2_EVENT_SNOOP_PKT_SENT);
0222 TX2_EVENT_ATTR(data_pktsent, CCPI2_EVENT_DATA_PKT_SENT);
0223 TX2_EVENT_ATTR(gic_pktsent, CCPI2_EVENT_GIC_PKT_SENT);
0224 
0225 static struct attribute *ccpi2_pmu_events_attrs[] = {
0226     &tx2_pmu_event_attr_req_pktsent.attr.attr,
0227     &tx2_pmu_event_attr_snoop_pktsent.attr.attr,
0228     &tx2_pmu_event_attr_data_pktsent.attr.attr,
0229     &tx2_pmu_event_attr_gic_pktsent.attr.attr,
0230     NULL,
0231 };
0232 
0233 static const struct attribute_group l3c_pmu_events_attr_group = {
0234     .name = "events",
0235     .attrs = l3c_pmu_events_attrs,
0236 };
0237 
0238 static const struct attribute_group dmc_pmu_events_attr_group = {
0239     .name = "events",
0240     .attrs = dmc_pmu_events_attrs,
0241 };
0242 
0243 static const struct attribute_group ccpi2_pmu_events_attr_group = {
0244     .name = "events",
0245     .attrs = ccpi2_pmu_events_attrs,
0246 };
0247 
0248 /*
0249  * sysfs cpumask attributes
0250  */
0251 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
0252         char *buf)
0253 {
0254     struct tx2_uncore_pmu *tx2_pmu;
0255 
0256     tx2_pmu = pmu_to_tx2_pmu(dev_get_drvdata(dev));
0257     return cpumap_print_to_pagebuf(true, buf, cpumask_of(tx2_pmu->cpu));
0258 }
0259 static DEVICE_ATTR_RO(cpumask);
0260 
0261 static struct attribute *tx2_pmu_cpumask_attrs[] = {
0262     &dev_attr_cpumask.attr,
0263     NULL,
0264 };
0265 
0266 static const struct attribute_group pmu_cpumask_attr_group = {
0267     .attrs = tx2_pmu_cpumask_attrs,
0268 };
0269 
0270 /*
0271  * Per PMU device attribute groups
0272  */
0273 static const struct attribute_group *l3c_pmu_attr_groups[] = {
0274     &l3c_pmu_format_attr_group,
0275     &pmu_cpumask_attr_group,
0276     &l3c_pmu_events_attr_group,
0277     NULL
0278 };
0279 
0280 static const struct attribute_group *dmc_pmu_attr_groups[] = {
0281     &dmc_pmu_format_attr_group,
0282     &pmu_cpumask_attr_group,
0283     &dmc_pmu_events_attr_group,
0284     NULL
0285 };
0286 
0287 static const struct attribute_group *ccpi2_pmu_attr_groups[] = {
0288     &ccpi2_pmu_format_attr_group,
0289     &pmu_cpumask_attr_group,
0290     &ccpi2_pmu_events_attr_group,
0291     NULL
0292 };
0293 
0294 static inline u32 reg_readl(unsigned long addr)
0295 {
0296     return readl((void __iomem *)addr);
0297 }
0298 
0299 static inline void reg_writel(u32 val, unsigned long addr)
0300 {
0301     writel(val, (void __iomem *)addr);
0302 }
0303 
0304 static int alloc_counter(struct tx2_uncore_pmu *tx2_pmu)
0305 {
0306     int counter;
0307 
0308     counter = find_first_zero_bit(tx2_pmu->active_counters,
0309                 tx2_pmu->max_counters);
0310     if (counter == tx2_pmu->max_counters)
0311         return -ENOSPC;
0312 
0313     set_bit(counter, tx2_pmu->active_counters);
0314     return counter;
0315 }
0316 
0317 static inline void free_counter(struct tx2_uncore_pmu *tx2_pmu, int counter)
0318 {
0319     clear_bit(counter, tx2_pmu->active_counters);
0320 }
0321 
0322 static void init_cntr_base_l3c(struct perf_event *event,
0323         struct tx2_uncore_pmu *tx2_pmu)
0324 {
0325     struct hw_perf_event *hwc = &event->hw;
0326     u32 cmask;
0327 
0328     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0329     cmask = tx2_pmu->counters_mask;
0330 
0331     /* counter ctrl/data reg offset at 8 */
0332     hwc->config_base = (unsigned long)tx2_pmu->base
0333         + L3C_COUNTER_CTL + (8 * GET_COUNTERID(event, cmask));
0334     hwc->event_base =  (unsigned long)tx2_pmu->base
0335         + L3C_COUNTER_DATA + (8 * GET_COUNTERID(event, cmask));
0336 }
0337 
0338 static void init_cntr_base_dmc(struct perf_event *event,
0339         struct tx2_uncore_pmu *tx2_pmu)
0340 {
0341     struct hw_perf_event *hwc = &event->hw;
0342     u32 cmask;
0343 
0344     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0345     cmask = tx2_pmu->counters_mask;
0346 
0347     hwc->config_base = (unsigned long)tx2_pmu->base
0348         + DMC_COUNTER_CTL;
0349     /* counter data reg offset at 0xc */
0350     hwc->event_base = (unsigned long)tx2_pmu->base
0351         + DMC_COUNTER_DATA + (0xc * GET_COUNTERID(event, cmask));
0352 }
0353 
0354 static void init_cntr_base_ccpi2(struct perf_event *event,
0355         struct tx2_uncore_pmu *tx2_pmu)
0356 {
0357     struct hw_perf_event *hwc = &event->hw;
0358     u32 cmask;
0359 
0360     cmask = tx2_pmu->counters_mask;
0361 
0362     hwc->config_base = (unsigned long)tx2_pmu->base
0363         + CCPI2_COUNTER_CTL + (4 * GET_COUNTERID(event, cmask));
0364     hwc->event_base =  (unsigned long)tx2_pmu->base;
0365 }
0366 
0367 static void uncore_start_event_l3c(struct perf_event *event, int flags)
0368 {
0369     u32 val, emask;
0370     struct hw_perf_event *hwc = &event->hw;
0371     struct tx2_uncore_pmu *tx2_pmu;
0372 
0373     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0374     emask = tx2_pmu->events_mask;
0375 
0376     /* event id encoded in bits [07:03] */
0377     val = GET_EVENTID(event, emask) << 3;
0378     reg_writel(val, hwc->config_base);
0379     local64_set(&hwc->prev_count, 0);
0380     reg_writel(0, hwc->event_base);
0381 }
0382 
0383 static inline void uncore_stop_event_l3c(struct perf_event *event)
0384 {
0385     reg_writel(0, event->hw.config_base);
0386 }
0387 
0388 static void uncore_start_event_dmc(struct perf_event *event, int flags)
0389 {
0390     u32 val, cmask, emask;
0391     struct hw_perf_event *hwc = &event->hw;
0392     struct tx2_uncore_pmu *tx2_pmu;
0393     int idx, event_id;
0394 
0395     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0396     cmask = tx2_pmu->counters_mask;
0397     emask = tx2_pmu->events_mask;
0398 
0399     idx = GET_COUNTERID(event, cmask);
0400     event_id = GET_EVENTID(event, emask);
0401 
0402     /* enable and start counters.
0403      * 8 bits for each counter, bits[05:01] of a counter to set event type.
0404      */
0405     val = reg_readl(hwc->config_base);
0406     val &= ~DMC_EVENT_CFG(idx, 0x1f);
0407     val |= DMC_EVENT_CFG(idx, event_id);
0408     reg_writel(val, hwc->config_base);
0409     local64_set(&hwc->prev_count, 0);
0410     reg_writel(0, hwc->event_base);
0411 }
0412 
0413 static void uncore_stop_event_dmc(struct perf_event *event)
0414 {
0415     u32 val, cmask;
0416     struct hw_perf_event *hwc = &event->hw;
0417     struct tx2_uncore_pmu *tx2_pmu;
0418     int idx;
0419 
0420     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0421     cmask = tx2_pmu->counters_mask;
0422     idx = GET_COUNTERID(event, cmask);
0423 
0424     /* clear event type(bits[05:01]) to stop counter */
0425     val = reg_readl(hwc->config_base);
0426     val &= ~DMC_EVENT_CFG(idx, 0x1f);
0427     reg_writel(val, hwc->config_base);
0428 }
0429 
0430 static void uncore_start_event_ccpi2(struct perf_event *event, int flags)
0431 {
0432     u32 emask;
0433     struct hw_perf_event *hwc = &event->hw;
0434     struct tx2_uncore_pmu *tx2_pmu;
0435 
0436     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0437     emask = tx2_pmu->events_mask;
0438 
0439     /* Bit [09:00] to set event id.
0440      * Bits [10], set level to rising edge.
0441      * Bits [11], set type to edge sensitive.
0442      */
0443     reg_writel((CCPI2_EVENT_TYPE_EDGE_SENSITIVE |
0444             CCPI2_EVENT_LEVEL_RISING_EDGE |
0445             GET_EVENTID(event, emask)), hwc->config_base);
0446 
0447     /* reset[4], enable[0] and start[1] counters */
0448     reg_writel(CCPI2_PERF_CTL_RESET |
0449             CCPI2_PERF_CTL_START |
0450             CCPI2_PERF_CTL_ENABLE,
0451             hwc->event_base + CCPI2_PERF_CTL);
0452     local64_set(&event->hw.prev_count, 0ULL);
0453 }
0454 
0455 static void uncore_stop_event_ccpi2(struct perf_event *event)
0456 {
0457     struct hw_perf_event *hwc = &event->hw;
0458 
0459     /* disable and stop counter */
0460     reg_writel(0, hwc->event_base + CCPI2_PERF_CTL);
0461 }
0462 
0463 static void tx2_uncore_event_update(struct perf_event *event)
0464 {
0465     u64 prev, delta, new = 0;
0466     struct hw_perf_event *hwc = &event->hw;
0467     struct tx2_uncore_pmu *tx2_pmu;
0468     enum tx2_uncore_type type;
0469     u32 prorate_factor;
0470     u32 cmask, emask;
0471 
0472     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0473     type = tx2_pmu->type;
0474     cmask = tx2_pmu->counters_mask;
0475     emask = tx2_pmu->events_mask;
0476     prorate_factor = tx2_pmu->prorate_factor;
0477     if (type == PMU_TYPE_CCPI2) {
0478         reg_writel(CCPI2_COUNTER_OFFSET +
0479                 GET_COUNTERID(event, cmask),
0480                 hwc->event_base + CCPI2_COUNTER_SEL);
0481         new = reg_readl(hwc->event_base + CCPI2_COUNTER_DATA_H);
0482         new = (new << 32) +
0483             reg_readl(hwc->event_base + CCPI2_COUNTER_DATA_L);
0484         prev = local64_xchg(&hwc->prev_count, new);
0485         delta = new - prev;
0486     } else {
0487         new = reg_readl(hwc->event_base);
0488         prev = local64_xchg(&hwc->prev_count, new);
0489         /* handles rollover of 32 bit counter */
0490         delta = (u32)(((1ULL << 32) - prev) + new);
0491     }
0492 
0493     /* DMC event data_transfers granularity is 16 Bytes, convert it to 64 */
0494     if (type == PMU_TYPE_DMC &&
0495             GET_EVENTID(event, emask) == DMC_EVENT_DATA_TRANSFERS)
0496         delta = delta/4;
0497 
0498     /* L3C and DMC has 16 and 8 interleave channels respectively.
0499      * The sampled value is for channel 0 and multiplied with
0500      * prorate_factor to get the count for a device.
0501      */
0502     local64_add(delta * prorate_factor, &event->count);
0503 }
0504 
0505 static enum tx2_uncore_type get_tx2_pmu_type(struct acpi_device *adev)
0506 {
0507     int i = 0;
0508     struct acpi_tx2_pmu_device {
0509         __u8 id[ACPI_ID_LEN];
0510         enum tx2_uncore_type type;
0511     } devices[] = {
0512         {"CAV901D", PMU_TYPE_L3C},
0513         {"CAV901F", PMU_TYPE_DMC},
0514         {"CAV901E", PMU_TYPE_CCPI2},
0515         {"", PMU_TYPE_INVALID}
0516     };
0517 
0518     while (devices[i].type != PMU_TYPE_INVALID) {
0519         if (!strcmp(acpi_device_hid(adev), devices[i].id))
0520             break;
0521         i++;
0522     }
0523 
0524     return devices[i].type;
0525 }
0526 
0527 static bool tx2_uncore_validate_event(struct pmu *pmu,
0528                   struct perf_event *event, int *counters)
0529 {
0530     if (is_software_event(event))
0531         return true;
0532     /* Reject groups spanning multiple HW PMUs. */
0533     if (event->pmu != pmu)
0534         return false;
0535 
0536     *counters = *counters + 1;
0537     return true;
0538 }
0539 
0540 /*
0541  * Make sure the group of events can be scheduled at once
0542  * on the PMU.
0543  */
0544 static bool tx2_uncore_validate_event_group(struct perf_event *event,
0545         int max_counters)
0546 {
0547     struct perf_event *sibling, *leader = event->group_leader;
0548     int counters = 0;
0549 
0550     if (event->group_leader == event)
0551         return true;
0552 
0553     if (!tx2_uncore_validate_event(event->pmu, leader, &counters))
0554         return false;
0555 
0556     for_each_sibling_event(sibling, leader) {
0557         if (!tx2_uncore_validate_event(event->pmu, sibling, &counters))
0558             return false;
0559     }
0560 
0561     if (!tx2_uncore_validate_event(event->pmu, event, &counters))
0562         return false;
0563 
0564     /*
0565      * If the group requires more counters than the HW has,
0566      * it cannot ever be scheduled.
0567      */
0568     return counters <= max_counters;
0569 }
0570 
0571 
0572 static int tx2_uncore_event_init(struct perf_event *event)
0573 {
0574     struct hw_perf_event *hwc = &event->hw;
0575     struct tx2_uncore_pmu *tx2_pmu;
0576 
0577     /* Test the event attr type check for PMU enumeration */
0578     if (event->attr.type != event->pmu->type)
0579         return -ENOENT;
0580 
0581     /*
0582      * SOC PMU counters are shared across all cores.
0583      * Therefore, it does not support per-process mode.
0584      * Also, it does not support event sampling mode.
0585      */
0586     if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
0587         return -EINVAL;
0588 
0589     if (event->cpu < 0)
0590         return -EINVAL;
0591 
0592     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0593     if (tx2_pmu->cpu >= nr_cpu_ids)
0594         return -EINVAL;
0595     event->cpu = tx2_pmu->cpu;
0596 
0597     if (event->attr.config >= tx2_pmu->max_events)
0598         return -EINVAL;
0599 
0600     /* store event id */
0601     hwc->config = event->attr.config;
0602 
0603     /* Validate the group */
0604     if (!tx2_uncore_validate_event_group(event, tx2_pmu->max_counters))
0605         return -EINVAL;
0606 
0607     return 0;
0608 }
0609 
0610 static void tx2_uncore_event_start(struct perf_event *event, int flags)
0611 {
0612     struct hw_perf_event *hwc = &event->hw;
0613     struct tx2_uncore_pmu *tx2_pmu;
0614 
0615     hwc->state = 0;
0616     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0617 
0618     tx2_pmu->start_event(event, flags);
0619     perf_event_update_userpage(event);
0620 
0621     /* No hrtimer needed for CCPI2, 64-bit counters */
0622     if (!tx2_pmu->hrtimer_callback)
0623         return;
0624 
0625     /* Start timer for first event */
0626     if (bitmap_weight(tx2_pmu->active_counters,
0627                 tx2_pmu->max_counters) == 1) {
0628         hrtimer_start(&tx2_pmu->hrtimer,
0629             ns_to_ktime(tx2_pmu->hrtimer_interval),
0630             HRTIMER_MODE_REL_PINNED);
0631     }
0632 }
0633 
0634 static void tx2_uncore_event_stop(struct perf_event *event, int flags)
0635 {
0636     struct hw_perf_event *hwc = &event->hw;
0637     struct tx2_uncore_pmu *tx2_pmu;
0638 
0639     if (hwc->state & PERF_HES_UPTODATE)
0640         return;
0641 
0642     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0643     tx2_pmu->stop_event(event);
0644     WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
0645     hwc->state |= PERF_HES_STOPPED;
0646     if (flags & PERF_EF_UPDATE) {
0647         tx2_uncore_event_update(event);
0648         hwc->state |= PERF_HES_UPTODATE;
0649     }
0650 }
0651 
0652 static int tx2_uncore_event_add(struct perf_event *event, int flags)
0653 {
0654     struct hw_perf_event *hwc = &event->hw;
0655     struct tx2_uncore_pmu *tx2_pmu;
0656 
0657     tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0658 
0659     /* Allocate a free counter */
0660     hwc->idx  = alloc_counter(tx2_pmu);
0661     if (hwc->idx < 0)
0662         return -EAGAIN;
0663 
0664     tx2_pmu->events[hwc->idx] = event;
0665     /* set counter control and data registers base address */
0666     tx2_pmu->init_cntr_base(event, tx2_pmu);
0667 
0668     hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
0669     if (flags & PERF_EF_START)
0670         tx2_uncore_event_start(event, flags);
0671 
0672     return 0;
0673 }
0674 
0675 static void tx2_uncore_event_del(struct perf_event *event, int flags)
0676 {
0677     struct tx2_uncore_pmu *tx2_pmu = pmu_to_tx2_pmu(event->pmu);
0678     struct hw_perf_event *hwc = &event->hw;
0679     u32 cmask;
0680 
0681     cmask = tx2_pmu->counters_mask;
0682     tx2_uncore_event_stop(event, PERF_EF_UPDATE);
0683 
0684     /* clear the assigned counter */
0685     free_counter(tx2_pmu, GET_COUNTERID(event, cmask));
0686 
0687     perf_event_update_userpage(event);
0688     tx2_pmu->events[hwc->idx] = NULL;
0689     hwc->idx = -1;
0690 
0691     if (!tx2_pmu->hrtimer_callback)
0692         return;
0693 
0694     if (bitmap_empty(tx2_pmu->active_counters, tx2_pmu->max_counters))
0695         hrtimer_cancel(&tx2_pmu->hrtimer);
0696 }
0697 
0698 static void tx2_uncore_event_read(struct perf_event *event)
0699 {
0700     tx2_uncore_event_update(event);
0701 }
0702 
0703 static enum hrtimer_restart tx2_hrtimer_callback(struct hrtimer *timer)
0704 {
0705     struct tx2_uncore_pmu *tx2_pmu;
0706     int max_counters, idx;
0707 
0708     tx2_pmu = container_of(timer, struct tx2_uncore_pmu, hrtimer);
0709     max_counters = tx2_pmu->max_counters;
0710 
0711     if (bitmap_empty(tx2_pmu->active_counters, max_counters))
0712         return HRTIMER_NORESTART;
0713 
0714     for_each_set_bit(idx, tx2_pmu->active_counters, max_counters) {
0715         struct perf_event *event = tx2_pmu->events[idx];
0716 
0717         tx2_uncore_event_update(event);
0718     }
0719     hrtimer_forward_now(timer, ns_to_ktime(tx2_pmu->hrtimer_interval));
0720     return HRTIMER_RESTART;
0721 }
0722 
0723 static int tx2_uncore_pmu_register(
0724         struct tx2_uncore_pmu *tx2_pmu)
0725 {
0726     struct device *dev = tx2_pmu->dev;
0727     char *name = tx2_pmu->name;
0728 
0729     /* Perf event registration */
0730     tx2_pmu->pmu = (struct pmu) {
0731         .module         = THIS_MODULE,
0732         .attr_groups    = tx2_pmu->attr_groups,
0733         .task_ctx_nr    = perf_invalid_context,
0734         .event_init = tx2_uncore_event_init,
0735         .add        = tx2_uncore_event_add,
0736         .del        = tx2_uncore_event_del,
0737         .start      = tx2_uncore_event_start,
0738         .stop       = tx2_uncore_event_stop,
0739         .read       = tx2_uncore_event_read,
0740         .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
0741     };
0742 
0743     tx2_pmu->pmu.name = devm_kasprintf(dev, GFP_KERNEL,
0744             "%s", name);
0745 
0746     return perf_pmu_register(&tx2_pmu->pmu, tx2_pmu->pmu.name, -1);
0747 }
0748 
0749 static int tx2_uncore_pmu_add_dev(struct tx2_uncore_pmu *tx2_pmu)
0750 {
0751     int ret, cpu;
0752 
0753     cpu = cpumask_any_and(cpumask_of_node(tx2_pmu->node),
0754             cpu_online_mask);
0755 
0756     tx2_pmu->cpu = cpu;
0757 
0758     if (tx2_pmu->hrtimer_callback) {
0759         hrtimer_init(&tx2_pmu->hrtimer,
0760                 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
0761         tx2_pmu->hrtimer.function = tx2_pmu->hrtimer_callback;
0762     }
0763 
0764     ret = tx2_uncore_pmu_register(tx2_pmu);
0765     if (ret) {
0766         dev_err(tx2_pmu->dev, "%s PMU: Failed to init driver\n",
0767                 tx2_pmu->name);
0768         return -ENODEV;
0769     }
0770 
0771     /* register hotplug callback for the pmu */
0772     ret = cpuhp_state_add_instance(
0773             CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE,
0774             &tx2_pmu->hpnode);
0775     if (ret) {
0776         dev_err(tx2_pmu->dev, "Error %d registering hotplug", ret);
0777         return ret;
0778     }
0779 
0780     /* Add to list */
0781     list_add(&tx2_pmu->entry, &tx2_pmus);
0782 
0783     dev_dbg(tx2_pmu->dev, "%s PMU UNCORE registered\n",
0784             tx2_pmu->pmu.name);
0785     return ret;
0786 }
0787 
0788 static struct tx2_uncore_pmu *tx2_uncore_pmu_init_dev(struct device *dev,
0789         acpi_handle handle, struct acpi_device *adev, u32 type)
0790 {
0791     struct tx2_uncore_pmu *tx2_pmu;
0792     void __iomem *base;
0793     struct resource res;
0794     struct resource_entry *rentry;
0795     struct list_head list;
0796     int ret;
0797 
0798     INIT_LIST_HEAD(&list);
0799     ret = acpi_dev_get_resources(adev, &list, NULL, NULL);
0800     if (ret <= 0) {
0801         dev_err(dev, "failed to parse _CRS method, error %d\n", ret);
0802         return NULL;
0803     }
0804 
0805     list_for_each_entry(rentry, &list, node) {
0806         if (resource_type(rentry->res) == IORESOURCE_MEM) {
0807             res = *rentry->res;
0808             rentry = NULL;
0809             break;
0810         }
0811     }
0812     acpi_dev_free_resource_list(&list);
0813 
0814     if (rentry) {
0815         dev_err(dev, "PMU type %d: Fail to find resource\n", type);
0816         return NULL;
0817     }
0818 
0819     base = devm_ioremap_resource(dev, &res);
0820     if (IS_ERR(base))
0821         return NULL;
0822 
0823     tx2_pmu = devm_kzalloc(dev, sizeof(*tx2_pmu), GFP_KERNEL);
0824     if (!tx2_pmu)
0825         return NULL;
0826 
0827     tx2_pmu->dev = dev;
0828     tx2_pmu->type = type;
0829     tx2_pmu->base = base;
0830     tx2_pmu->node = dev_to_node(dev);
0831     INIT_LIST_HEAD(&tx2_pmu->entry);
0832 
0833     switch (tx2_pmu->type) {
0834     case PMU_TYPE_L3C:
0835         tx2_pmu->max_counters = TX2_PMU_DMC_L3C_MAX_COUNTERS;
0836         tx2_pmu->counters_mask = 0x3;
0837         tx2_pmu->prorate_factor = TX2_PMU_L3_TILES;
0838         tx2_pmu->max_events = L3_EVENT_MAX;
0839         tx2_pmu->events_mask = 0x1f;
0840         tx2_pmu->hrtimer_interval = TX2_PMU_HRTIMER_INTERVAL;
0841         tx2_pmu->hrtimer_callback = tx2_hrtimer_callback;
0842         tx2_pmu->attr_groups = l3c_pmu_attr_groups;
0843         tx2_pmu->name = devm_kasprintf(dev, GFP_KERNEL,
0844                 "uncore_l3c_%d", tx2_pmu->node);
0845         tx2_pmu->init_cntr_base = init_cntr_base_l3c;
0846         tx2_pmu->start_event = uncore_start_event_l3c;
0847         tx2_pmu->stop_event = uncore_stop_event_l3c;
0848         break;
0849     case PMU_TYPE_DMC:
0850         tx2_pmu->max_counters = TX2_PMU_DMC_L3C_MAX_COUNTERS;
0851         tx2_pmu->counters_mask = 0x3;
0852         tx2_pmu->prorate_factor = TX2_PMU_DMC_CHANNELS;
0853         tx2_pmu->max_events = DMC_EVENT_MAX;
0854         tx2_pmu->events_mask = 0x1f;
0855         tx2_pmu->hrtimer_interval = TX2_PMU_HRTIMER_INTERVAL;
0856         tx2_pmu->hrtimer_callback = tx2_hrtimer_callback;
0857         tx2_pmu->attr_groups = dmc_pmu_attr_groups;
0858         tx2_pmu->name = devm_kasprintf(dev, GFP_KERNEL,
0859                 "uncore_dmc_%d", tx2_pmu->node);
0860         tx2_pmu->init_cntr_base = init_cntr_base_dmc;
0861         tx2_pmu->start_event = uncore_start_event_dmc;
0862         tx2_pmu->stop_event = uncore_stop_event_dmc;
0863         break;
0864     case PMU_TYPE_CCPI2:
0865         /* CCPI2 has 8 counters */
0866         tx2_pmu->max_counters = TX2_PMU_CCPI2_MAX_COUNTERS;
0867         tx2_pmu->counters_mask = 0x7;
0868         tx2_pmu->prorate_factor = 1;
0869         tx2_pmu->max_events = CCPI2_EVENT_MAX;
0870         tx2_pmu->events_mask = 0x1ff;
0871         tx2_pmu->attr_groups = ccpi2_pmu_attr_groups;
0872         tx2_pmu->name = devm_kasprintf(dev, GFP_KERNEL,
0873                 "uncore_ccpi2_%d", tx2_pmu->node);
0874         tx2_pmu->init_cntr_base = init_cntr_base_ccpi2;
0875         tx2_pmu->start_event = uncore_start_event_ccpi2;
0876         tx2_pmu->stop_event = uncore_stop_event_ccpi2;
0877         tx2_pmu->hrtimer_callback = NULL;
0878         break;
0879     case PMU_TYPE_INVALID:
0880         devm_kfree(dev, tx2_pmu);
0881         return NULL;
0882     }
0883 
0884     return tx2_pmu;
0885 }
0886 
0887 static acpi_status tx2_uncore_pmu_add(acpi_handle handle, u32 level,
0888                     void *data, void **return_value)
0889 {
0890     struct acpi_device *adev = acpi_fetch_acpi_dev(handle);
0891     struct tx2_uncore_pmu *tx2_pmu;
0892     enum tx2_uncore_type type;
0893 
0894     if (!adev || acpi_bus_get_status(adev) || !adev->status.present)
0895         return AE_OK;
0896 
0897     type = get_tx2_pmu_type(adev);
0898     if (type == PMU_TYPE_INVALID)
0899         return AE_OK;
0900 
0901     tx2_pmu = tx2_uncore_pmu_init_dev((struct device *)data,
0902             handle, adev, type);
0903 
0904     if (!tx2_pmu)
0905         return AE_ERROR;
0906 
0907     if (tx2_uncore_pmu_add_dev(tx2_pmu)) {
0908         /* Can't add the PMU device, abort */
0909         return AE_ERROR;
0910     }
0911     return AE_OK;
0912 }
0913 
0914 static int tx2_uncore_pmu_online_cpu(unsigned int cpu,
0915         struct hlist_node *hpnode)
0916 {
0917     struct tx2_uncore_pmu *tx2_pmu;
0918 
0919     tx2_pmu = hlist_entry_safe(hpnode,
0920             struct tx2_uncore_pmu, hpnode);
0921 
0922     /* Pick this CPU, If there is no CPU/PMU association and both are
0923      * from same node.
0924      */
0925     if ((tx2_pmu->cpu >= nr_cpu_ids) &&
0926         (tx2_pmu->node == cpu_to_node(cpu)))
0927         tx2_pmu->cpu = cpu;
0928 
0929     return 0;
0930 }
0931 
0932 static int tx2_uncore_pmu_offline_cpu(unsigned int cpu,
0933         struct hlist_node *hpnode)
0934 {
0935     int new_cpu;
0936     struct tx2_uncore_pmu *tx2_pmu;
0937     struct cpumask cpu_online_mask_temp;
0938 
0939     tx2_pmu = hlist_entry_safe(hpnode,
0940             struct tx2_uncore_pmu, hpnode);
0941 
0942     if (cpu != tx2_pmu->cpu)
0943         return 0;
0944 
0945     if (tx2_pmu->hrtimer_callback)
0946         hrtimer_cancel(&tx2_pmu->hrtimer);
0947 
0948     cpumask_copy(&cpu_online_mask_temp, cpu_online_mask);
0949     cpumask_clear_cpu(cpu, &cpu_online_mask_temp);
0950     new_cpu = cpumask_any_and(
0951             cpumask_of_node(tx2_pmu->node),
0952             &cpu_online_mask_temp);
0953 
0954     tx2_pmu->cpu = new_cpu;
0955     if (new_cpu >= nr_cpu_ids)
0956         return 0;
0957     perf_pmu_migrate_context(&tx2_pmu->pmu, cpu, new_cpu);
0958 
0959     return 0;
0960 }
0961 
0962 static const struct acpi_device_id tx2_uncore_acpi_match[] = {
0963     {"CAV901C", 0},
0964     {},
0965 };
0966 MODULE_DEVICE_TABLE(acpi, tx2_uncore_acpi_match);
0967 
0968 static int tx2_uncore_probe(struct platform_device *pdev)
0969 {
0970     struct device *dev = &pdev->dev;
0971     acpi_handle handle;
0972     acpi_status status;
0973 
0974     set_dev_node(dev, acpi_get_node(ACPI_HANDLE(dev)));
0975 
0976     if (!has_acpi_companion(dev))
0977         return -ENODEV;
0978 
0979     handle = ACPI_HANDLE(dev);
0980     if (!handle)
0981         return -EINVAL;
0982 
0983     /* Walk through the tree for all PMU UNCORE devices */
0984     status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1,
0985                      tx2_uncore_pmu_add,
0986                      NULL, dev, NULL);
0987     if (ACPI_FAILURE(status)) {
0988         dev_err(dev, "failed to probe PMU devices\n");
0989         return_ACPI_STATUS(status);
0990     }
0991 
0992     dev_info(dev, "node%d: pmu uncore registered\n", dev_to_node(dev));
0993     return 0;
0994 }
0995 
0996 static int tx2_uncore_remove(struct platform_device *pdev)
0997 {
0998     struct tx2_uncore_pmu *tx2_pmu, *temp;
0999     struct device *dev = &pdev->dev;
1000 
1001     if (!list_empty(&tx2_pmus)) {
1002         list_for_each_entry_safe(tx2_pmu, temp, &tx2_pmus, entry) {
1003             if (tx2_pmu->node == dev_to_node(dev)) {
1004                 cpuhp_state_remove_instance_nocalls(
1005                     CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE,
1006                     &tx2_pmu->hpnode);
1007                 perf_pmu_unregister(&tx2_pmu->pmu);
1008                 list_del(&tx2_pmu->entry);
1009             }
1010         }
1011     }
1012     return 0;
1013 }
1014 
1015 static struct platform_driver tx2_uncore_driver = {
1016     .driver = {
1017         .name       = "tx2-uncore-pmu",
1018         .acpi_match_table = ACPI_PTR(tx2_uncore_acpi_match),
1019         .suppress_bind_attrs = true,
1020     },
1021     .probe = tx2_uncore_probe,
1022     .remove = tx2_uncore_remove,
1023 };
1024 
1025 static int __init tx2_uncore_driver_init(void)
1026 {
1027     int ret;
1028 
1029     ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE,
1030                       "perf/tx2/uncore:online",
1031                       tx2_uncore_pmu_online_cpu,
1032                       tx2_uncore_pmu_offline_cpu);
1033     if (ret) {
1034         pr_err("TX2 PMU: setup hotplug failed(%d)\n", ret);
1035         return ret;
1036     }
1037     ret = platform_driver_register(&tx2_uncore_driver);
1038     if (ret)
1039         cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE);
1040 
1041     return ret;
1042 }
1043 module_init(tx2_uncore_driver_init);
1044 
1045 static void __exit tx2_uncore_driver_exit(void)
1046 {
1047     platform_driver_unregister(&tx2_uncore_driver);
1048     cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CAVIUM_TX2_UNCORE_ONLINE);
1049 }
1050 module_exit(tx2_uncore_driver_exit);
1051 
1052 MODULE_DESCRIPTION("ThunderX2 UNCORE PMU driver");
1053 MODULE_LICENSE("GPL v2");
1054 MODULE_AUTHOR("Ganapatrao Kulkarni <gkulkarni@cavium.com>");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!