OSCL-LXR linux-6.0.1/​arch/​arm/​mm/​cache-l2x0-pmu.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * L220/L310 cache controller support
  *
  * Copyright (C) 2016 ARM Limited
  */
 #include <linux/errno.h>
 #include <linux/hrtimer.h>
 #include <linux/io.h>
 #include <linux/list.h>
 #include <linux/perf_event.h>
 #include <linux/printk.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 
 #include <asm/hardware/cache-l2x0.h>
 
 #define PMU_NR_COUNTERS 2
 
 static void __iomem *l2x0_base;
 static struct pmu *l2x0_pmu;
 static cpumask_t pmu_cpu;
 
 static const char *l2x0_name;
 
 static ktime_t l2x0_pmu_poll_period;
 static struct hrtimer l2x0_pmu_hrtimer;
 
 /*
  * The L220/PL310 PMU has two equivalent counters, Counter1 and Counter0.
  * Registers controlling these are laid out in pairs, in descending order, i.e.
  * the register for Counter1 comes first, followed by the register for
  * Counter0.
  * We ensure that idx 0 -> Counter0, and idx1 -> Counter1.
  */
 static struct perf_event *events[PMU_NR_COUNTERS];
 
 /* Find an unused counter */
 static int l2x0_pmu_find_idx(void)
 {
     int i;
 
     for (i = 0; i < PMU_NR_COUNTERS; i++) {
         if (!events[i])
             return i;
     }
 
     return -1;
 }
 
 /* How many counters are allocated? */
 static int l2x0_pmu_num_active_counters(void)
 {
     int i, cnt = 0;
 
     for (i = 0; i < PMU_NR_COUNTERS; i++) {
         if (events[i])
             cnt++;
     }
 
     return cnt;
 }
 
 static void l2x0_pmu_counter_config_write(int idx, u32 val)
 {
     writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_CFG - 4 * idx);
 }
 
 static u32 l2x0_pmu_counter_read(int idx)
 {
     return readl_relaxed(l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
 }
 
 static void l2x0_pmu_counter_write(int idx, u32 val)
 {
     writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx);
 }
 
 static void __l2x0_pmu_enable(void)
 {
     u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
     val |= L2X0_EVENT_CNT_CTRL_ENABLE;
     writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
 }
 
 static void __l2x0_pmu_disable(void)
 {
     u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL);
     val &= ~L2X0_EVENT_CNT_CTRL_ENABLE;
     writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL);
 }
 
 static void l2x0_pmu_enable(struct pmu *pmu)
 {
     if (l2x0_pmu_num_active_counters() == 0)
         return;
 
     __l2x0_pmu_enable();
 }
 
 static void l2x0_pmu_disable(struct pmu *pmu)
 {
     if (l2x0_pmu_num_active_counters() == 0)
         return;
 
     __l2x0_pmu_disable();
 }
 
 static void warn_if_saturated(u32 count)
 {
     if (count != 0xffffffff)
         return;
 
     pr_warn_ratelimited("L2X0 counter saturated. Poll period too long\n");
 }
 
 static void l2x0_pmu_event_read(struct perf_event *event)
 {
     struct hw_perf_event *hw = &event->hw;
     u64 prev_count, new_count, mask;
 
     do {
          prev_count = local64_read(&hw->prev_count);
          new_count = l2x0_pmu_counter_read(hw->idx);
     } while (local64_xchg(&hw->prev_count, new_count) != prev_count);
 
     mask = GENMASK_ULL(31, 0);
     local64_add((new_count - prev_count) & mask, &event->count);
 
     warn_if_saturated(new_count);
 }
 
 static void l2x0_pmu_event_configure(struct perf_event *event)
 {
     struct hw_perf_event *hw = &event->hw;
 
     /*
      * The L2X0 counters saturate at 0xffffffff rather than wrapping, so we
      * will *always* lose some number of events when a counter saturates,
      * and have no way of detecting how many were lost.
      *
      * To minimize the impact of this, we try to maximize the period by
      * always starting counters at zero. To ensure that group ratios are
      * representative, we poll periodically to avoid counters saturating.
      * See l2x0_pmu_poll().
      */
     local64_set(&hw->prev_count, 0);
     l2x0_pmu_counter_write(hw->idx, 0);
 }
 
 static enum hrtimer_restart l2x0_pmu_poll(struct hrtimer *hrtimer)
 {
     unsigned long flags;
     int i;
 
     local_irq_save(flags);
     __l2x0_pmu_disable();
 
     for (i = 0; i < PMU_NR_COUNTERS; i++) {
         struct perf_event *event = events[i];
 
         if (!event)
             continue;
 
         l2x0_pmu_event_read(event);
         l2x0_pmu_event_configure(event);
     }
 
     __l2x0_pmu_enable();
     local_irq_restore(flags);
 
     hrtimer_forward_now(hrtimer, l2x0_pmu_poll_period);
     return HRTIMER_RESTART;
 }
 
 
 static void __l2x0_pmu_event_enable(int idx, u32 event)
 {
     u32 val;
 
     val = event << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
     val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
     l2x0_pmu_counter_config_write(idx, val);
 }
 
 static void l2x0_pmu_event_start(struct perf_event *event, int flags)
 {
     struct hw_perf_event *hw = &event->hw;
 
     if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
         return;
 
     if (flags & PERF_EF_RELOAD) {
         WARN_ON_ONCE(!(hw->state & PERF_HES_UPTODATE));
         l2x0_pmu_event_configure(event);
     }
 
     hw->state = 0;
 
     __l2x0_pmu_event_enable(hw->idx, hw->config_base);
 }
 
 static void __l2x0_pmu_event_disable(int idx)
 {
     u32 val;
 
     val = L2X0_EVENT_CNT_CFG_SRC_DISABLED << L2X0_EVENT_CNT_CFG_SRC_SHIFT;
     val |= L2X0_EVENT_CNT_CFG_INT_DISABLED;
     l2x0_pmu_counter_config_write(idx, val);
 }
 
 static void l2x0_pmu_event_stop(struct perf_event *event, int flags)
 {
     struct hw_perf_event *hw = &event->hw;
 
     if (WARN_ON_ONCE(event->hw.state & PERF_HES_STOPPED))
         return;
 
     __l2x0_pmu_event_disable(hw->idx);
 
     hw->state |= PERF_HES_STOPPED;
 
     if (flags & PERF_EF_UPDATE) {
         l2x0_pmu_event_read(event);
         hw->state |= PERF_HES_UPTODATE;
     }
 }
 
 static int l2x0_pmu_event_add(struct perf_event *event, int flags)
 {
     struct hw_perf_event *hw = &event->hw;
     int idx = l2x0_pmu_find_idx();
 
     if (idx == -1)
         return -EAGAIN;
 
     /*
      * Pin the timer, so that the overflows are handled by the chosen
      * event->cpu (this is the same one as presented in "cpumask"
      * attribute).
      */
     if (l2x0_pmu_num_active_counters() == 0)
         hrtimer_start(&l2x0_pmu_hrtimer, l2x0_pmu_poll_period,
                   HRTIMER_MODE_REL_PINNED);
 
     events[idx] = event;
     hw->idx = idx;
 
     l2x0_pmu_event_configure(event);
 
     hw->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 
     if (flags & PERF_EF_START)
         l2x0_pmu_event_start(event, 0);
 
     return 0;
 }
 
 static void l2x0_pmu_event_del(struct perf_event *event, int flags)
 {
     struct hw_perf_event *hw = &event->hw;
 
     l2x0_pmu_event_stop(event, PERF_EF_UPDATE);
 
     events[hw->idx] = NULL;
     hw->idx = -1;
 
     if (l2x0_pmu_num_active_counters() == 0)
         hrtimer_cancel(&l2x0_pmu_hrtimer);
 }
 
 static bool l2x0_pmu_group_is_valid(struct perf_event *event)
 {
     struct pmu *pmu = event->pmu;
     struct perf_event *leader = event->group_leader;
     struct perf_event *sibling;
     int num_hw = 0;
 
     if (leader->pmu == pmu)
         num_hw++;
     else if (!is_software_event(leader))
         return false;
 
     for_each_sibling_event(sibling, leader) {
         if (sibling->pmu == pmu)
             num_hw++;
         else if (!is_software_event(sibling))
             return false;
     }
 
     return num_hw <= PMU_NR_COUNTERS;
 }
 
 static int l2x0_pmu_event_init(struct perf_event *event)
 {
     struct hw_perf_event *hw = &event->hw;
 
     if (event->attr.type != l2x0_pmu->type)
         return -ENOENT;
 
     if (is_sampling_event(event) ||
         event->attach_state & PERF_ATTACH_TASK)
         return -EINVAL;
 
     if (event->cpu < 0)
         return -EINVAL;
 
     if (event->attr.config & ~L2X0_EVENT_CNT_CFG_SRC_MASK)
         return -EINVAL;
 
     hw->config_base = event->attr.config;
 
     if (!l2x0_pmu_group_is_valid(event))
         return -EINVAL;
 
     event->cpu = cpumask_first(&pmu_cpu);
 
     return 0;
 }
 
 struct l2x0_event_attribute {
     struct device_attribute attr;
     unsigned int config;
     bool pl310_only;
 };
 
 #define L2X0_EVENT_ATTR(_name, _config, _pl310_only)                \
     (&((struct l2x0_event_attribute[]) {{                   \
         .attr = __ATTR(_name, S_IRUGO, l2x0_pmu_event_show, NULL),  \
         .config = _config,                      \
         .pl310_only = _pl310_only,                  \
     }})[0].attr.attr)
 
 #define L220_PLUS_EVENT_ATTR(_name, _config)                    \
     L2X0_EVENT_ATTR(_name, _config, false)
 
 #define PL310_EVENT_ATTR(_name, _config)                    \
     L2X0_EVENT_ATTR(_name, _config, true)
 
 static ssize_t l2x0_pmu_event_show(struct device *dev,
                    struct device_attribute *attr, char *buf)
 {
     struct l2x0_event_attribute *lattr;
 
     lattr = container_of(attr, typeof(*lattr), attr);
     return snprintf(buf, PAGE_SIZE, "config=0x%x\n", lattr->config);
 }
 
 static umode_t l2x0_pmu_event_attr_is_visible(struct kobject *kobj,
                           struct attribute *attr,
                           int unused)
 {
     struct device *dev = kobj_to_dev(kobj);
     struct pmu *pmu = dev_get_drvdata(dev);
     struct l2x0_event_attribute *lattr;
 
     lattr = container_of(attr, typeof(*lattr), attr.attr);
 
     if (!lattr->pl310_only || strcmp("l2c_310", pmu->name) == 0)
         return attr->mode;
 
     return 0;
 }
 
 static struct attribute *l2x0_pmu_event_attrs[] = {
     L220_PLUS_EVENT_ATTR(co,    0x1),
     L220_PLUS_EVENT_ATTR(drhit, 0x2),
     L220_PLUS_EVENT_ATTR(drreq, 0x3),
     L220_PLUS_EVENT_ATTR(dwhit, 0x4),
     L220_PLUS_EVENT_ATTR(dwreq, 0x5),
     L220_PLUS_EVENT_ATTR(dwtreq,    0x6),
     L220_PLUS_EVENT_ATTR(irhit, 0x7),
     L220_PLUS_EVENT_ATTR(irreq, 0x8),
     L220_PLUS_EVENT_ATTR(wa,    0x9),
     PL310_EVENT_ATTR(ipfalloc,  0xa),
     PL310_EVENT_ATTR(epfhit,    0xb),
     PL310_EVENT_ATTR(epfalloc,  0xc),
     PL310_EVENT_ATTR(srrcvd,    0xd),
     PL310_EVENT_ATTR(srconf,    0xe),
     PL310_EVENT_ATTR(epfrcvd,   0xf),
     NULL
 };
 
 static struct attribute_group l2x0_pmu_event_attrs_group = {
     .name = "events",
     .attrs = l2x0_pmu_event_attrs,
     .is_visible = l2x0_pmu_event_attr_is_visible,
 };
 
 static ssize_t l2x0_pmu_cpumask_show(struct device *dev,
                      struct device_attribute *attr, char *buf)
 {
     return cpumap_print_to_pagebuf(true, buf, &pmu_cpu);
 }
 
 static struct device_attribute l2x0_pmu_cpumask_attr =
         __ATTR(cpumask, S_IRUGO, l2x0_pmu_cpumask_show, NULL);
 
 static struct attribute *l2x0_pmu_cpumask_attrs[] = {
     &l2x0_pmu_cpumask_attr.attr,
     NULL,
 };
 
 static struct attribute_group l2x0_pmu_cpumask_attr_group = {
     .attrs = l2x0_pmu_cpumask_attrs,
 };
 
 static const struct attribute_group *l2x0_pmu_attr_groups[] = {
     &l2x0_pmu_event_attrs_group,
     &l2x0_pmu_cpumask_attr_group,
     NULL,
 };
 
 static void l2x0_pmu_reset(void)
 {
     int i;
 
     __l2x0_pmu_disable();
 
     for (i = 0; i < PMU_NR_COUNTERS; i++)
         __l2x0_pmu_event_disable(i);
 }
 
 static int l2x0_pmu_offline_cpu(unsigned int cpu)
 {
     unsigned int target;
 
     if (!cpumask_test_and_clear_cpu(cpu, &pmu_cpu))
         return 0;
 
     target = cpumask_any_but(cpu_online_mask, cpu);
     if (target >= nr_cpu_ids)
         return 0;
 
     perf_pmu_migrate_context(l2x0_pmu, cpu, target);
     cpumask_set_cpu(target, &pmu_cpu);
 
     return 0;
 }
 
 void l2x0_pmu_suspend(void)
 {
     int i;
 
     if (!l2x0_pmu)
         return;
 
     l2x0_pmu_disable(l2x0_pmu);
 
     for (i = 0; i < PMU_NR_COUNTERS; i++) {
         if (events[i])
             l2x0_pmu_event_stop(events[i], PERF_EF_UPDATE);
     }
 
 }
 
 void l2x0_pmu_resume(void)
 {
     int i;
 
     if (!l2x0_pmu)
         return;
 
     l2x0_pmu_reset();
 
     for (i = 0; i < PMU_NR_COUNTERS; i++) {
         if (events[i])
             l2x0_pmu_event_start(events[i], PERF_EF_RELOAD);
     }
 
     l2x0_pmu_enable(l2x0_pmu);
 }
 
 void __init l2x0_pmu_register(void __iomem *base, u32 part)
 {
     /*
      * Determine whether we support the PMU, and choose the name for sysfs.
      * This is also used by l2x0_pmu_event_attr_is_visible to determine
      * which events to display, as the PL310 PMU supports a superset of
      * L220 events.
      *
      * The L210 PMU has a different programmer's interface, and is not
      * supported by this driver.
      *
      * We must defer registering the PMU until the perf subsystem is up and
      * running, so just stash the name and base, and leave that to another
      * initcall.
      */
     switch (part & L2X0_CACHE_ID_PART_MASK) {
     case L2X0_CACHE_ID_PART_L220:
         l2x0_name = "l2c_220";
         break;
     case L2X0_CACHE_ID_PART_L310:
         l2x0_name = "l2c_310";
         break;
     default:
         return;
     }
 
     l2x0_base = base;
 }
 
 static __init int l2x0_pmu_init(void)
 {
     int ret;
 
     if (!l2x0_base)
         return 0;
 
     l2x0_pmu = kzalloc(sizeof(*l2x0_pmu), GFP_KERNEL);
     if (!l2x0_pmu) {
         pr_warn("Unable to allocate L2x0 PMU\n");
         return -ENOMEM;
     }
 
     *l2x0_pmu = (struct pmu) {
         .task_ctx_nr = perf_invalid_context,
         .pmu_enable = l2x0_pmu_enable,
         .pmu_disable = l2x0_pmu_disable,
         .read = l2x0_pmu_event_read,
         .start = l2x0_pmu_event_start,
         .stop = l2x0_pmu_event_stop,
         .add = l2x0_pmu_event_add,
         .del = l2x0_pmu_event_del,
         .event_init = l2x0_pmu_event_init,
         .attr_groups = l2x0_pmu_attr_groups,
         .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
     };
 
     l2x0_pmu_reset();
 
     /*
      * We always use a hrtimer rather than an interrupt.
      * See comments in l2x0_pmu_event_configure and l2x0_pmu_poll.
      *
      * Polling once a second allows the counters to fill up to 1/128th on a
      * quad-core test chip with cores clocked at 400MHz. Hopefully this
      * leaves sufficient headroom to avoid overflow on production silicon
      * at higher frequencies.
      */
     l2x0_pmu_poll_period = ms_to_ktime(1000);
     hrtimer_init(&l2x0_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
     l2x0_pmu_hrtimer.function = l2x0_pmu_poll;
 
     cpumask_set_cpu(0, &pmu_cpu);
     ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE,
                     "perf/arm/l2x0:online", NULL,
                     l2x0_pmu_offline_cpu);
     if (ret)
         goto out_pmu;
 
     ret = perf_pmu_register(l2x0_pmu, l2x0_name, -1);
     if (ret)
         goto out_cpuhp;
 
     return 0;
 
 out_cpuhp:
     cpuhp_remove_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE);
 out_pmu:
     kfree(l2x0_pmu);
     l2x0_pmu = NULL;
     return ret;
 }
 device_initcall(l2x0_pmu_init);

This page was automatically generated by the 2.1.0 LXR engine. The LXR team