Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2013 Advanced Micro Devices, Inc.
0004  *
0005  * Author: Jacob Shin <jacob.shin@amd.com>
0006  */
0007 
0008 #include <linux/perf_event.h>
0009 #include <linux/percpu.h>
0010 #include <linux/types.h>
0011 #include <linux/slab.h>
0012 #include <linux/init.h>
0013 #include <linux/cpu.h>
0014 #include <linux/cpumask.h>
0015 #include <linux/cpufeature.h>
0016 #include <linux/smp.h>
0017 
0018 #include <asm/perf_event.h>
0019 #include <asm/msr.h>
0020 
0021 #define NUM_COUNTERS_NB     4
0022 #define NUM_COUNTERS_L2     4
0023 #define NUM_COUNTERS_L3     6
0024 
0025 #define RDPMC_BASE_NB       6
0026 #define RDPMC_BASE_LLC      10
0027 
0028 #define COUNTER_SHIFT       16
0029 
0030 #undef pr_fmt
0031 #define pr_fmt(fmt) "amd_uncore: " fmt
0032 
0033 static int pmu_version;
0034 static int num_counters_llc;
0035 static int num_counters_nb;
0036 static bool l3_mask;
0037 
0038 static HLIST_HEAD(uncore_unused_list);
0039 
0040 struct amd_uncore {
0041     int id;
0042     int refcnt;
0043     int cpu;
0044     int num_counters;
0045     int rdpmc_base;
0046     u32 msr_base;
0047     cpumask_t *active_mask;
0048     struct pmu *pmu;
0049     struct perf_event **events;
0050     struct hlist_node node;
0051 };
0052 
0053 static struct amd_uncore * __percpu *amd_uncore_nb;
0054 static struct amd_uncore * __percpu *amd_uncore_llc;
0055 
0056 static struct pmu amd_nb_pmu;
0057 static struct pmu amd_llc_pmu;
0058 
0059 static cpumask_t amd_nb_active_mask;
0060 static cpumask_t amd_llc_active_mask;
0061 
0062 static bool is_nb_event(struct perf_event *event)
0063 {
0064     return event->pmu->type == amd_nb_pmu.type;
0065 }
0066 
0067 static bool is_llc_event(struct perf_event *event)
0068 {
0069     return event->pmu->type == amd_llc_pmu.type;
0070 }
0071 
0072 static struct amd_uncore *event_to_amd_uncore(struct perf_event *event)
0073 {
0074     if (is_nb_event(event) && amd_uncore_nb)
0075         return *per_cpu_ptr(amd_uncore_nb, event->cpu);
0076     else if (is_llc_event(event) && amd_uncore_llc)
0077         return *per_cpu_ptr(amd_uncore_llc, event->cpu);
0078 
0079     return NULL;
0080 }
0081 
0082 static void amd_uncore_read(struct perf_event *event)
0083 {
0084     struct hw_perf_event *hwc = &event->hw;
0085     u64 prev, new;
0086     s64 delta;
0087 
0088     /*
0089      * since we do not enable counter overflow interrupts,
0090      * we do not have to worry about prev_count changing on us
0091      */
0092 
0093     prev = local64_read(&hwc->prev_count);
0094     rdpmcl(hwc->event_base_rdpmc, new);
0095     local64_set(&hwc->prev_count, new);
0096     delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
0097     delta >>= COUNTER_SHIFT;
0098     local64_add(delta, &event->count);
0099 }
0100 
0101 static void amd_uncore_start(struct perf_event *event, int flags)
0102 {
0103     struct hw_perf_event *hwc = &event->hw;
0104 
0105     if (flags & PERF_EF_RELOAD)
0106         wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
0107 
0108     hwc->state = 0;
0109     wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
0110     perf_event_update_userpage(event);
0111 }
0112 
0113 static void amd_uncore_stop(struct perf_event *event, int flags)
0114 {
0115     struct hw_perf_event *hwc = &event->hw;
0116 
0117     wrmsrl(hwc->config_base, hwc->config);
0118     hwc->state |= PERF_HES_STOPPED;
0119 
0120     if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
0121         amd_uncore_read(event);
0122         hwc->state |= PERF_HES_UPTODATE;
0123     }
0124 }
0125 
0126 static int amd_uncore_add(struct perf_event *event, int flags)
0127 {
0128     int i;
0129     struct amd_uncore *uncore = event_to_amd_uncore(event);
0130     struct hw_perf_event *hwc = &event->hw;
0131 
0132     /* are we already assigned? */
0133     if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
0134         goto out;
0135 
0136     for (i = 0; i < uncore->num_counters; i++) {
0137         if (uncore->events[i] == event) {
0138             hwc->idx = i;
0139             goto out;
0140         }
0141     }
0142 
0143     /* if not, take the first available counter */
0144     hwc->idx = -1;
0145     for (i = 0; i < uncore->num_counters; i++) {
0146         if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
0147             hwc->idx = i;
0148             break;
0149         }
0150     }
0151 
0152 out:
0153     if (hwc->idx == -1)
0154         return -EBUSY;
0155 
0156     hwc->config_base = uncore->msr_base + (2 * hwc->idx);
0157     hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
0158     hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
0159     hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
0160 
0161     /*
0162      * The first four DF counters are accessible via RDPMC index 6 to 9
0163      * followed by the L3 counters from index 10 to 15. For processors
0164      * with more than four DF counters, the DF RDPMC assignments become
0165      * discontiguous as the additional counters are accessible starting
0166      * from index 16.
0167      */
0168     if (is_nb_event(event) && hwc->idx >= NUM_COUNTERS_NB)
0169         hwc->event_base_rdpmc += NUM_COUNTERS_L3;
0170 
0171     if (flags & PERF_EF_START)
0172         amd_uncore_start(event, PERF_EF_RELOAD);
0173 
0174     return 0;
0175 }
0176 
0177 static void amd_uncore_del(struct perf_event *event, int flags)
0178 {
0179     int i;
0180     struct amd_uncore *uncore = event_to_amd_uncore(event);
0181     struct hw_perf_event *hwc = &event->hw;
0182 
0183     amd_uncore_stop(event, PERF_EF_UPDATE);
0184 
0185     for (i = 0; i < uncore->num_counters; i++) {
0186         if (cmpxchg(&uncore->events[i], event, NULL) == event)
0187             break;
0188     }
0189 
0190     hwc->idx = -1;
0191 }
0192 
0193 /*
0194  * Return a full thread and slice mask unless user
0195  * has provided them
0196  */
0197 static u64 l3_thread_slice_mask(u64 config)
0198 {
0199     if (boot_cpu_data.x86 <= 0x18)
0200         return ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) |
0201                ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK);
0202 
0203     /*
0204      * If the user doesn't specify a threadmask, they're not trying to
0205      * count core 0, so we enable all cores & threads.
0206      * We'll also assume that they want to count slice 0 if they specify
0207      * a threadmask and leave sliceid and enallslices unpopulated.
0208      */
0209     if (!(config & AMD64_L3_F19H_THREAD_MASK))
0210         return AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES |
0211                AMD64_L3_EN_ALL_CORES;
0212 
0213     return config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK |
0214              AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES |
0215              AMD64_L3_COREID_MASK);
0216 }
0217 
0218 static int amd_uncore_event_init(struct perf_event *event)
0219 {
0220     struct amd_uncore *uncore;
0221     struct hw_perf_event *hwc = &event->hw;
0222     u64 event_mask = AMD64_RAW_EVENT_MASK_NB;
0223 
0224     if (event->attr.type != event->pmu->type)
0225         return -ENOENT;
0226 
0227     if (pmu_version >= 2 && is_nb_event(event))
0228         event_mask = AMD64_PERFMON_V2_RAW_EVENT_MASK_NB;
0229 
0230     /*
0231      * NB and Last level cache counters (MSRs) are shared across all cores
0232      * that share the same NB / Last level cache.  On family 16h and below,
0233      * Interrupts can be directed to a single target core, however, event
0234      * counts generated by processes running on other cores cannot be masked
0235      * out. So we do not support sampling and per-thread events via
0236      * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts:
0237      */
0238     hwc->config = event->attr.config & event_mask;
0239     hwc->idx = -1;
0240 
0241     if (event->cpu < 0)
0242         return -EINVAL;
0243 
0244     /*
0245      * SliceMask and ThreadMask need to be set for certain L3 events.
0246      * For other events, the two fields do not affect the count.
0247      */
0248     if (l3_mask && is_llc_event(event))
0249         hwc->config |= l3_thread_slice_mask(event->attr.config);
0250 
0251     uncore = event_to_amd_uncore(event);
0252     if (!uncore)
0253         return -ENODEV;
0254 
0255     /*
0256      * since request can come in to any of the shared cores, we will remap
0257      * to a single common cpu.
0258      */
0259     event->cpu = uncore->cpu;
0260 
0261     return 0;
0262 }
0263 
0264 static umode_t
0265 amd_f17h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i)
0266 {
0267     return boot_cpu_data.x86 >= 0x17 && boot_cpu_data.x86 < 0x19 ?
0268            attr->mode : 0;
0269 }
0270 
0271 static umode_t
0272 amd_f19h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i)
0273 {
0274     return boot_cpu_data.x86 >= 0x19 ? attr->mode : 0;
0275 }
0276 
0277 static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
0278                         struct device_attribute *attr,
0279                         char *buf)
0280 {
0281     cpumask_t *active_mask;
0282     struct pmu *pmu = dev_get_drvdata(dev);
0283 
0284     if (pmu->type == amd_nb_pmu.type)
0285         active_mask = &amd_nb_active_mask;
0286     else if (pmu->type == amd_llc_pmu.type)
0287         active_mask = &amd_llc_active_mask;
0288     else
0289         return 0;
0290 
0291     return cpumap_print_to_pagebuf(true, buf, active_mask);
0292 }
0293 static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
0294 
0295 static struct attribute *amd_uncore_attrs[] = {
0296     &dev_attr_cpumask.attr,
0297     NULL,
0298 };
0299 
0300 static struct attribute_group amd_uncore_attr_group = {
0301     .attrs = amd_uncore_attrs,
0302 };
0303 
0304 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format)         \
0305 static ssize_t __uncore_##_var##_show(struct device *dev,       \
0306                 struct device_attribute *attr,      \
0307                 char *page)             \
0308 {                                   \
0309     BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);         \
0310     return sprintf(page, _format "\n");             \
0311 }                                   \
0312 static struct device_attribute format_attr_##_var =         \
0313     __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
0314 
0315 DEFINE_UNCORE_FORMAT_ATTR(event12,  event,      "config:0-7,32-35");
0316 DEFINE_UNCORE_FORMAT_ATTR(event14,  event,      "config:0-7,32-35,59-60"); /* F17h+ DF */
0317 DEFINE_UNCORE_FORMAT_ATTR(event14v2,    event,      "config:0-7,32-37");       /* PerfMonV2 DF */
0318 DEFINE_UNCORE_FORMAT_ATTR(event8,   event,      "config:0-7");         /* F17h+ L3 */
0319 DEFINE_UNCORE_FORMAT_ATTR(umask8,   umask,      "config:8-15");
0320 DEFINE_UNCORE_FORMAT_ATTR(umask12,  umask,      "config:8-15,24-27");      /* PerfMonV2 DF */
0321 DEFINE_UNCORE_FORMAT_ATTR(coreid,   coreid,     "config:42-44");       /* F19h L3 */
0322 DEFINE_UNCORE_FORMAT_ATTR(slicemask,    slicemask,  "config:48-51");       /* F17h L3 */
0323 DEFINE_UNCORE_FORMAT_ATTR(threadmask8,  threadmask, "config:56-63");       /* F17h L3 */
0324 DEFINE_UNCORE_FORMAT_ATTR(threadmask2,  threadmask, "config:56-57");       /* F19h L3 */
0325 DEFINE_UNCORE_FORMAT_ATTR(enallslices,  enallslices,    "config:46");          /* F19h L3 */
0326 DEFINE_UNCORE_FORMAT_ATTR(enallcores,   enallcores, "config:47");          /* F19h L3 */
0327 DEFINE_UNCORE_FORMAT_ATTR(sliceid,  sliceid,    "config:48-50");       /* F19h L3 */
0328 
0329 /* Common DF and NB attributes */
0330 static struct attribute *amd_uncore_df_format_attr[] = {
0331     &format_attr_event12.attr,  /* event */
0332     &format_attr_umask8.attr,   /* umask */
0333     NULL,
0334 };
0335 
0336 /* Common L2 and L3 attributes */
0337 static struct attribute *amd_uncore_l3_format_attr[] = {
0338     &format_attr_event12.attr,  /* event */
0339     &format_attr_umask8.attr,   /* umask */
0340     NULL,               /* threadmask */
0341     NULL,
0342 };
0343 
0344 /* F17h unique L3 attributes */
0345 static struct attribute *amd_f17h_uncore_l3_format_attr[] = {
0346     &format_attr_slicemask.attr,    /* slicemask */
0347     NULL,
0348 };
0349 
0350 /* F19h unique L3 attributes */
0351 static struct attribute *amd_f19h_uncore_l3_format_attr[] = {
0352     &format_attr_coreid.attr,   /* coreid */
0353     &format_attr_enallslices.attr,  /* enallslices */
0354     &format_attr_enallcores.attr,   /* enallcores */
0355     &format_attr_sliceid.attr,  /* sliceid */
0356     NULL,
0357 };
0358 
0359 static struct attribute_group amd_uncore_df_format_group = {
0360     .name = "format",
0361     .attrs = amd_uncore_df_format_attr,
0362 };
0363 
0364 static struct attribute_group amd_uncore_l3_format_group = {
0365     .name = "format",
0366     .attrs = amd_uncore_l3_format_attr,
0367 };
0368 
0369 static struct attribute_group amd_f17h_uncore_l3_format_group = {
0370     .name = "format",
0371     .attrs = amd_f17h_uncore_l3_format_attr,
0372     .is_visible = amd_f17h_uncore_is_visible,
0373 };
0374 
0375 static struct attribute_group amd_f19h_uncore_l3_format_group = {
0376     .name = "format",
0377     .attrs = amd_f19h_uncore_l3_format_attr,
0378     .is_visible = amd_f19h_uncore_is_visible,
0379 };
0380 
0381 static const struct attribute_group *amd_uncore_df_attr_groups[] = {
0382     &amd_uncore_attr_group,
0383     &amd_uncore_df_format_group,
0384     NULL,
0385 };
0386 
0387 static const struct attribute_group *amd_uncore_l3_attr_groups[] = {
0388     &amd_uncore_attr_group,
0389     &amd_uncore_l3_format_group,
0390     NULL,
0391 };
0392 
0393 static const struct attribute_group *amd_uncore_l3_attr_update[] = {
0394     &amd_f17h_uncore_l3_format_group,
0395     &amd_f19h_uncore_l3_format_group,
0396     NULL,
0397 };
0398 
0399 static struct pmu amd_nb_pmu = {
0400     .task_ctx_nr    = perf_invalid_context,
0401     .attr_groups    = amd_uncore_df_attr_groups,
0402     .name       = "amd_nb",
0403     .event_init = amd_uncore_event_init,
0404     .add        = amd_uncore_add,
0405     .del        = amd_uncore_del,
0406     .start      = amd_uncore_start,
0407     .stop       = amd_uncore_stop,
0408     .read       = amd_uncore_read,
0409     .capabilities   = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
0410     .module     = THIS_MODULE,
0411 };
0412 
0413 static struct pmu amd_llc_pmu = {
0414     .task_ctx_nr    = perf_invalid_context,
0415     .attr_groups    = amd_uncore_l3_attr_groups,
0416     .attr_update    = amd_uncore_l3_attr_update,
0417     .name       = "amd_l2",
0418     .event_init = amd_uncore_event_init,
0419     .add        = amd_uncore_add,
0420     .del        = amd_uncore_del,
0421     .start      = amd_uncore_start,
0422     .stop       = amd_uncore_stop,
0423     .read       = amd_uncore_read,
0424     .capabilities   = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
0425     .module     = THIS_MODULE,
0426 };
0427 
0428 static struct amd_uncore *amd_uncore_alloc(unsigned int cpu)
0429 {
0430     return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
0431             cpu_to_node(cpu));
0432 }
0433 
0434 static inline struct perf_event **
0435 amd_uncore_events_alloc(unsigned int num, unsigned int cpu)
0436 {
0437     return kzalloc_node(sizeof(struct perf_event *) * num, GFP_KERNEL,
0438                 cpu_to_node(cpu));
0439 }
0440 
0441 static int amd_uncore_cpu_up_prepare(unsigned int cpu)
0442 {
0443     struct amd_uncore *uncore_nb = NULL, *uncore_llc = NULL;
0444 
0445     if (amd_uncore_nb) {
0446         *per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
0447         uncore_nb = amd_uncore_alloc(cpu);
0448         if (!uncore_nb)
0449             goto fail;
0450         uncore_nb->cpu = cpu;
0451         uncore_nb->num_counters = num_counters_nb;
0452         uncore_nb->rdpmc_base = RDPMC_BASE_NB;
0453         uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
0454         uncore_nb->active_mask = &amd_nb_active_mask;
0455         uncore_nb->pmu = &amd_nb_pmu;
0456         uncore_nb->events = amd_uncore_events_alloc(num_counters_nb, cpu);
0457         if (!uncore_nb->events)
0458             goto fail;
0459         uncore_nb->id = -1;
0460         *per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
0461     }
0462 
0463     if (amd_uncore_llc) {
0464         *per_cpu_ptr(amd_uncore_llc, cpu) = NULL;
0465         uncore_llc = amd_uncore_alloc(cpu);
0466         if (!uncore_llc)
0467             goto fail;
0468         uncore_llc->cpu = cpu;
0469         uncore_llc->num_counters = num_counters_llc;
0470         uncore_llc->rdpmc_base = RDPMC_BASE_LLC;
0471         uncore_llc->msr_base = MSR_F16H_L2I_PERF_CTL;
0472         uncore_llc->active_mask = &amd_llc_active_mask;
0473         uncore_llc->pmu = &amd_llc_pmu;
0474         uncore_llc->events = amd_uncore_events_alloc(num_counters_llc, cpu);
0475         if (!uncore_llc->events)
0476             goto fail;
0477         uncore_llc->id = -1;
0478         *per_cpu_ptr(amd_uncore_llc, cpu) = uncore_llc;
0479     }
0480 
0481     return 0;
0482 
0483 fail:
0484     if (uncore_nb) {
0485         kfree(uncore_nb->events);
0486         kfree(uncore_nb);
0487     }
0488 
0489     if (uncore_llc) {
0490         kfree(uncore_llc->events);
0491         kfree(uncore_llc);
0492     }
0493 
0494     return -ENOMEM;
0495 }
0496 
0497 static struct amd_uncore *
0498 amd_uncore_find_online_sibling(struct amd_uncore *this,
0499                    struct amd_uncore * __percpu *uncores)
0500 {
0501     unsigned int cpu;
0502     struct amd_uncore *that;
0503 
0504     for_each_online_cpu(cpu) {
0505         that = *per_cpu_ptr(uncores, cpu);
0506 
0507         if (!that)
0508             continue;
0509 
0510         if (this == that)
0511             continue;
0512 
0513         if (this->id == that->id) {
0514             hlist_add_head(&this->node, &uncore_unused_list);
0515             this = that;
0516             break;
0517         }
0518     }
0519 
0520     this->refcnt++;
0521     return this;
0522 }
0523 
0524 static int amd_uncore_cpu_starting(unsigned int cpu)
0525 {
0526     unsigned int eax, ebx, ecx, edx;
0527     struct amd_uncore *uncore;
0528 
0529     if (amd_uncore_nb) {
0530         uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
0531         cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
0532         uncore->id = ecx & 0xff;
0533 
0534         uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
0535         *per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
0536     }
0537 
0538     if (amd_uncore_llc) {
0539         uncore = *per_cpu_ptr(amd_uncore_llc, cpu);
0540         uncore->id = get_llc_id(cpu);
0541 
0542         uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_llc);
0543         *per_cpu_ptr(amd_uncore_llc, cpu) = uncore;
0544     }
0545 
0546     return 0;
0547 }
0548 
0549 static void uncore_clean_online(void)
0550 {
0551     struct amd_uncore *uncore;
0552     struct hlist_node *n;
0553 
0554     hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) {
0555         hlist_del(&uncore->node);
0556         kfree(uncore);
0557     }
0558 }
0559 
0560 static void uncore_online(unsigned int cpu,
0561               struct amd_uncore * __percpu *uncores)
0562 {
0563     struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);
0564 
0565     uncore_clean_online();
0566 
0567     if (cpu == uncore->cpu)
0568         cpumask_set_cpu(cpu, uncore->active_mask);
0569 }
0570 
0571 static int amd_uncore_cpu_online(unsigned int cpu)
0572 {
0573     if (amd_uncore_nb)
0574         uncore_online(cpu, amd_uncore_nb);
0575 
0576     if (amd_uncore_llc)
0577         uncore_online(cpu, amd_uncore_llc);
0578 
0579     return 0;
0580 }
0581 
0582 static void uncore_down_prepare(unsigned int cpu,
0583                 struct amd_uncore * __percpu *uncores)
0584 {
0585     unsigned int i;
0586     struct amd_uncore *this = *per_cpu_ptr(uncores, cpu);
0587 
0588     if (this->cpu != cpu)
0589         return;
0590 
0591     /* this cpu is going down, migrate to a shared sibling if possible */
0592     for_each_online_cpu(i) {
0593         struct amd_uncore *that = *per_cpu_ptr(uncores, i);
0594 
0595         if (cpu == i)
0596             continue;
0597 
0598         if (this == that) {
0599             perf_pmu_migrate_context(this->pmu, cpu, i);
0600             cpumask_clear_cpu(cpu, that->active_mask);
0601             cpumask_set_cpu(i, that->active_mask);
0602             that->cpu = i;
0603             break;
0604         }
0605     }
0606 }
0607 
0608 static int amd_uncore_cpu_down_prepare(unsigned int cpu)
0609 {
0610     if (amd_uncore_nb)
0611         uncore_down_prepare(cpu, amd_uncore_nb);
0612 
0613     if (amd_uncore_llc)
0614         uncore_down_prepare(cpu, amd_uncore_llc);
0615 
0616     return 0;
0617 }
0618 
0619 static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores)
0620 {
0621     struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);
0622 
0623     if (cpu == uncore->cpu)
0624         cpumask_clear_cpu(cpu, uncore->active_mask);
0625 
0626     if (!--uncore->refcnt) {
0627         kfree(uncore->events);
0628         kfree(uncore);
0629     }
0630 
0631     *per_cpu_ptr(uncores, cpu) = NULL;
0632 }
0633 
0634 static int amd_uncore_cpu_dead(unsigned int cpu)
0635 {
0636     if (amd_uncore_nb)
0637         uncore_dead(cpu, amd_uncore_nb);
0638 
0639     if (amd_uncore_llc)
0640         uncore_dead(cpu, amd_uncore_llc);
0641 
0642     return 0;
0643 }
0644 
0645 static int __init amd_uncore_init(void)
0646 {
0647     struct attribute **df_attr = amd_uncore_df_format_attr;
0648     struct attribute **l3_attr = amd_uncore_l3_format_attr;
0649     union cpuid_0x80000022_ebx ebx;
0650     int ret = -ENODEV;
0651 
0652     if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
0653         boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
0654         return -ENODEV;
0655 
0656     if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
0657         return -ENODEV;
0658 
0659     if (boot_cpu_has(X86_FEATURE_PERFMON_V2))
0660         pmu_version = 2;
0661 
0662     num_counters_nb = NUM_COUNTERS_NB;
0663     num_counters_llc = NUM_COUNTERS_L2;
0664     if (boot_cpu_data.x86 >= 0x17) {
0665         /*
0666          * For F17h and above, the Northbridge counters are
0667          * repurposed as Data Fabric counters. Also, L3
0668          * counters are supported too. The PMUs are exported
0669          * based on family as either L2 or L3 and NB or DF.
0670          */
0671         num_counters_llc      = NUM_COUNTERS_L3;
0672         amd_nb_pmu.name       = "amd_df";
0673         amd_llc_pmu.name      = "amd_l3";
0674         l3_mask           = true;
0675     }
0676 
0677     if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
0678         if (pmu_version >= 2) {
0679             *df_attr++ = &format_attr_event14v2.attr;
0680             *df_attr++ = &format_attr_umask12.attr;
0681         } else if (boot_cpu_data.x86 >= 0x17) {
0682             *df_attr = &format_attr_event14.attr;
0683         }
0684 
0685         amd_uncore_nb = alloc_percpu(struct amd_uncore *);
0686         if (!amd_uncore_nb) {
0687             ret = -ENOMEM;
0688             goto fail_nb;
0689         }
0690         ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
0691         if (ret)
0692             goto fail_nb;
0693 
0694         if (pmu_version >= 2) {
0695             ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
0696             num_counters_nb = ebx.split.num_df_pmc;
0697         }
0698 
0699         pr_info("%d %s %s counters detected\n", num_counters_nb,
0700             boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ?  "HYGON" : "",
0701             amd_nb_pmu.name);
0702 
0703         ret = 0;
0704     }
0705 
0706     if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) {
0707         if (boot_cpu_data.x86 >= 0x19) {
0708             *l3_attr++ = &format_attr_event8.attr;
0709             *l3_attr++ = &format_attr_umask8.attr;
0710             *l3_attr++ = &format_attr_threadmask2.attr;
0711         } else if (boot_cpu_data.x86 >= 0x17) {
0712             *l3_attr++ = &format_attr_event8.attr;
0713             *l3_attr++ = &format_attr_umask8.attr;
0714             *l3_attr++ = &format_attr_threadmask8.attr;
0715         }
0716 
0717         amd_uncore_llc = alloc_percpu(struct amd_uncore *);
0718         if (!amd_uncore_llc) {
0719             ret = -ENOMEM;
0720             goto fail_llc;
0721         }
0722         ret = perf_pmu_register(&amd_llc_pmu, amd_llc_pmu.name, -1);
0723         if (ret)
0724             goto fail_llc;
0725 
0726         pr_info("%d %s %s counters detected\n", num_counters_llc,
0727             boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ?  "HYGON" : "",
0728             amd_llc_pmu.name);
0729         ret = 0;
0730     }
0731 
0732     /*
0733      * Install callbacks. Core will call them for each online cpu.
0734      */
0735     if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
0736                   "perf/x86/amd/uncore:prepare",
0737                   amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead))
0738         goto fail_llc;
0739 
0740     if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
0741                   "perf/x86/amd/uncore:starting",
0742                   amd_uncore_cpu_starting, NULL))
0743         goto fail_prep;
0744     if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
0745                   "perf/x86/amd/uncore:online",
0746                   amd_uncore_cpu_online,
0747                   amd_uncore_cpu_down_prepare))
0748         goto fail_start;
0749     return 0;
0750 
0751 fail_start:
0752     cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
0753 fail_prep:
0754     cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
0755 fail_llc:
0756     if (boot_cpu_has(X86_FEATURE_PERFCTR_NB))
0757         perf_pmu_unregister(&amd_nb_pmu);
0758     free_percpu(amd_uncore_llc);
0759 fail_nb:
0760     free_percpu(amd_uncore_nb);
0761 
0762     return ret;
0763 }
0764 
0765 static void __exit amd_uncore_exit(void)
0766 {
0767     cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE);
0768     cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
0769     cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
0770 
0771     if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) {
0772         perf_pmu_unregister(&amd_llc_pmu);
0773         free_percpu(amd_uncore_llc);
0774         amd_uncore_llc = NULL;
0775     }
0776 
0777     if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
0778         perf_pmu_unregister(&amd_nb_pmu);
0779         free_percpu(amd_uncore_nb);
0780         amd_uncore_nb = NULL;
0781     }
0782 }
0783 
0784 module_init(amd_uncore_init);
0785 module_exit(amd_uncore_exit);
0786 
0787 MODULE_DESCRIPTION("AMD Uncore Driver");
0788 MODULE_LICENSE("GPL v2");
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!