events/amd/brs.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Implement support for AMD Fam19h Branch Sampling feature
0004  * Based on specifications published in AMD PPR Fam19 Model 01
0005  *
0006  * Copyright 2021 Google LLC
0007  * Contributed by Stephane Eranian <eranian@google.com>
0008  */
0009 #include <linux/kernel.h>
0010 #include <linux/jump_label.h>
0011 #include <asm/msr.h>
0012 #include <asm/cpufeature.h>
0013
0014 #include "../perf_event.h"
0015
0016 #define BRS_POISON  0xFFFFFFFFFFFFFFFEULL /* mark limit of valid entries */
0017
0018 /* Debug Extension Configuration register layout */
0019 union amd_debug_extn_cfg {
0020     __u64 val;
0021     struct {
0022         __u64   rsvd0:2,  /* reserved */
0023             brsmen:1, /* branch sample enable */
0024             rsvd4_3:2,/* reserved - must be 0x3 */
0025             vb:1,     /* valid branches recorded */
0026             rsvd2:10, /* reserved */
0027             msroff:4, /* index of next entry to write */
0028             rsvd3:4,  /* reserved */
0029             pmc:3,    /* #PMC holding the sampling event */
0030             rsvd4:37; /* reserved */
0031     };
0032 };
0033
0034 static inline unsigned int brs_from(int idx)
0035 {
0036     return MSR_AMD_SAMP_BR_FROM + 2 * idx;
0037 }
0038
0039 static inline unsigned int brs_to(int idx)
0040 {
0041     return MSR_AMD_SAMP_BR_FROM + 2 * idx + 1;
0042 }
0043
0044 static inline void set_debug_extn_cfg(u64 val)
0045 {
0046     /* bits[4:3] must always be set to 11b */
0047     wrmsrl(MSR_AMD_DBG_EXTN_CFG, val | 3ULL << 3);
0048 }
0049
0050 static inline u64 get_debug_extn_cfg(void)
0051 {
0052     u64 val;
0053
0054     rdmsrl(MSR_AMD_DBG_EXTN_CFG, val);
0055     return val;
0056 }
0057
0058 static bool __init amd_brs_detect(void)
0059 {
0060     if (!cpu_feature_enabled(X86_FEATURE_BRS))
0061         return false;
0062
0063     switch (boot_cpu_data.x86) {
0064     case 0x19: /* AMD Fam19h (Zen3) */
0065         x86_pmu.lbr_nr = 16;
0066
0067         /* No hardware filtering supported */
0068         x86_pmu.lbr_sel_map = NULL;
0069         x86_pmu.lbr_sel_mask = 0;
0070         break;
0071     default:
0072         return false;
0073     }
0074
0075     return true;
0076 }
0077
0078 /*
0079  * Current BRS implementation does not support branch type or privilege level
0080  * filtering. Therefore, this function simply enforces these limitations. No need for
0081  * a br_sel_map. Software filtering is not supported because it would not correlate well
0082  * with a sampling period.
0083  */
0084 int amd_brs_setup_filter(struct perf_event *event)
0085 {
0086     u64 type = event->attr.branch_sample_type;
0087
0088     /* No BRS support */
0089     if (!x86_pmu.lbr_nr)
0090         return -EOPNOTSUPP;
0091
0092     /* Can only capture all branches, i.e., no filtering */
0093     if ((type & ~PERF_SAMPLE_BRANCH_PLM_ALL) != PERF_SAMPLE_BRANCH_ANY)
0094         return -EINVAL;
0095
0096     return 0;
0097 }
0098
0099 /* tos = top of stack, i.e., last valid entry written */
0100 static inline int amd_brs_get_tos(union amd_debug_extn_cfg *cfg)
0101 {
0102     /*
0103      * msroff: index of next entry to write so top-of-stack is one off
0104      * if BRS is full then msroff is set back to 0.
0105      */
0106     return (cfg->msroff ? cfg->msroff : x86_pmu.lbr_nr) - 1;
0107 }
0108
0109 /*
0110  * make sure we have a sane BRS offset to begin with
0111  * especially with kexec
0112  */
0113 void amd_brs_reset(void)
0114 {
0115     if (!cpu_feature_enabled(X86_FEATURE_BRS))
0116         return;
0117
0118     /*
0119      * Reset config
0120      */
0121     set_debug_extn_cfg(0);
0122
0123     /*
0124      * Mark first entry as poisoned
0125      */
0126     wrmsrl(brs_to(0), BRS_POISON);
0127 }
0128
0129 int __init amd_brs_init(void)
0130 {
0131     if (!amd_brs_detect())
0132         return -EOPNOTSUPP;
0133
0134     pr_cont("%d-deep BRS, ", x86_pmu.lbr_nr);
0135
0136     return 0;
0137 }
0138
0139 void amd_brs_enable(void)
0140 {
0141     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0142     union amd_debug_extn_cfg cfg;
0143
0144     /* Activate only on first user */
0145     if (++cpuc->brs_active > 1)
0146         return;
0147
0148     cfg.val    = 0; /* reset all fields */
0149     cfg.brsmen = 1; /* enable branch sampling */
0150
0151     /* Set enable bit */
0152     set_debug_extn_cfg(cfg.val);
0153 }
0154
0155 void amd_brs_enable_all(void)
0156 {
0157     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0158     if (cpuc->lbr_users)
0159         amd_brs_enable();
0160 }
0161
0162 void amd_brs_disable(void)
0163 {
0164     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0165     union amd_debug_extn_cfg cfg;
0166
0167     /* Check if active (could be disabled via x86_pmu_disable_all()) */
0168     if (!cpuc->brs_active)
0169         return;
0170
0171     /* Only disable for last user */
0172     if (--cpuc->brs_active)
0173         return;
0174
0175     /*
0176      * Clear the brsmen bit but preserve the others as they contain
0177      * useful state such as vb and msroff
0178      */
0179     cfg.val = get_debug_extn_cfg();
0180
0181     /*
0182      * When coming in on interrupt and BRS is full, then hw will have
0183      * already stopped BRS, no need to issue wrmsr again
0184      */
0185     if (cfg.brsmen) {
0186         cfg.brsmen = 0;
0187         set_debug_extn_cfg(cfg.val);
0188     }
0189 }
0190
0191 void amd_brs_disable_all(void)
0192 {
0193     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0194     if (cpuc->lbr_users)
0195         amd_brs_disable();
0196 }
0197
0198 static bool amd_brs_match_plm(struct perf_event *event, u64 to)
0199 {
0200     int type = event->attr.branch_sample_type;
0201     int plm_k = PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_HV;
0202     int plm_u = PERF_SAMPLE_BRANCH_USER;
0203
0204     if (!(type & plm_k) && kernel_ip(to))
0205         return 0;
0206
0207     if (!(type & plm_u) && !kernel_ip(to))
0208         return 0;
0209
0210     return 1;
0211 }
0212
0213 /*
0214  * Caller must ensure amd_brs_inuse() is true before calling
0215  * return:
0216  */
0217 void amd_brs_drain(void)
0218 {
0219     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0220     struct perf_event *event = cpuc->events[0];
0221     struct perf_branch_entry *br = cpuc->lbr_entries;
0222     union amd_debug_extn_cfg cfg;
0223     u32 i, nr = 0, num, tos, start;
0224     u32 shift = 64 - boot_cpu_data.x86_virt_bits;
0225
0226     /*
0227      * BRS event forced on PMC0,
0228      * so check if there is an event.
0229      * It is possible to have lbr_users > 0 but the event
0230      * not yet scheduled due to long latency PMU irq
0231      */
0232     if (!event)
0233         goto empty;
0234
0235     cfg.val = get_debug_extn_cfg();
0236
0237     /* Sanity check [0-x86_pmu.lbr_nr] */
0238     if (WARN_ON_ONCE(cfg.msroff >= x86_pmu.lbr_nr))
0239         goto empty;
0240
0241     /* No valid branch */
0242     if (cfg.vb == 0)
0243         goto empty;
0244
0245     /*
0246      * msr.off points to next entry to be written
0247      * tos = most recent entry index = msr.off - 1
0248      * BRS register buffer saturates, so we know we have
0249      * start < tos and that we have to read from start to tos
0250      */
0251     start = 0;
0252     tos = amd_brs_get_tos(&cfg);
0253
0254     num = tos - start + 1;
0255
0256     /*
0257      * BRS is only one pass (saturation) from MSROFF to depth-1
0258      * MSROFF wraps to zero when buffer is full
0259      */
0260     for (i = 0; i < num; i++) {
0261         u32 brs_idx = tos - i;
0262         u64 from, to;
0263
0264         rdmsrl(brs_to(brs_idx), to);
0265
0266         /* Entry does not belong to us (as marked by kernel) */
0267         if (to == BRS_POISON)
0268             break;
0269
0270         /*
0271          * Sign-extend SAMP_BR_TO to 64 bits, bits 61-63 are reserved.
0272          * Necessary to generate proper virtual addresses suitable for
0273          * symbolization
0274          */
0275         to = (u64)(((s64)to << shift) >> shift);
0276
0277         if (!amd_brs_match_plm(event, to))
0278             continue;
0279
0280         rdmsrl(brs_from(brs_idx), from);
0281
0282         perf_clear_branch_entry_bitfields(br+nr);
0283
0284         br[nr].from = from;
0285         br[nr].to   = to;
0286
0287         nr++;
0288     }
0289 empty:
0290     /* Record number of sampled branches */
0291     cpuc->lbr_stack.nr = nr;
0292 }
0293
0294 /*
0295  * Poison most recent entry to prevent reuse by next task
0296  * required because BRS entry are not tagged by PID
0297  */
0298 static void amd_brs_poison_buffer(void)
0299 {
0300     union amd_debug_extn_cfg cfg;
0301     unsigned int idx;
0302
0303     /* Get current state */
0304     cfg.val = get_debug_extn_cfg();
0305
0306     /* idx is most recently written entry */
0307     idx = amd_brs_get_tos(&cfg);
0308
0309     /* Poison target of entry */
0310     wrmsrl(brs_to(idx), BRS_POISON);
0311 }
0312
0313 /*
0314  * On context switch in, we need to make sure no samples from previous user
0315  * are left in the BRS.
0316  *
0317  * On ctxswin, sched_in = true, called after the PMU has started
0318  * On ctxswout, sched_in = false, called before the PMU is stopped
0319  */
0320 void amd_pmu_brs_sched_task(struct perf_event_context *ctx, bool sched_in)
0321 {
0322     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0323
0324     /* no active users */
0325     if (!cpuc->lbr_users)
0326         return;
0327
0328     /*
0329      * On context switch in, we need to ensure we do not use entries
0330      * from previous BRS user on that CPU, so we poison the buffer as
0331      * a faster way compared to resetting all entries.
0332      */
0333     if (sched_in)
0334         amd_brs_poison_buffer();
0335 }
0336
0337 /*
0338  * called from ACPI processor_idle.c or acpi_pad.c
0339  * with interrupts disabled
0340  */
0341 void perf_amd_brs_lopwr_cb(bool lopwr_in)
0342 {
0343     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0344     union amd_debug_extn_cfg cfg;
0345
0346     /*
0347      * on mwait in, we may end up in non C0 state.
0348      * we must disable branch sampling to avoid holding the NMI
0349      * for too long. We disable it in hardware but we
0350      * keep the state in cpuc, so we can re-enable.
0351      *
0352      * The hardware will deliver the NMI if needed when brsmen cleared
0353      */
0354     if (cpuc->brs_active) {
0355         cfg.val = get_debug_extn_cfg();
0356         cfg.brsmen = !lopwr_in;
0357         set_debug_extn_cfg(cfg.val);
0358     }
0359 }
0360
0361 DEFINE_STATIC_CALL_NULL(perf_lopwr_cb, perf_amd_brs_lopwr_cb);
0362 EXPORT_STATIC_CALL_TRAMP_GPL(perf_lopwr_cb);
0363
0364 void __init amd_brs_lopwr_init(void)
0365 {
0366     static_call_update(perf_lopwr_cb, perf_amd_brs_lopwr_cb);
0367 }