Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * BTS PMU driver for perf
0004  * Copyright (c) 2013-2014, Intel Corporation.
0005  */
0006 
0007 #undef DEBUG
0008 
0009 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0010 
0011 #include <linux/bitops.h>
0012 #include <linux/types.h>
0013 #include <linux/slab.h>
0014 #include <linux/debugfs.h>
0015 #include <linux/device.h>
0016 #include <linux/coredump.h>
0017 
0018 #include <linux/sizes.h>
0019 #include <asm/perf_event.h>
0020 
0021 #include "../perf_event.h"
0022 
0023 struct bts_ctx {
0024     struct perf_output_handle   handle;
0025     struct debug_store      ds_back;
0026     int             state;
0027 };
0028 
0029 /* BTS context states: */
0030 enum {
0031     /* no ongoing AUX transactions */
0032     BTS_STATE_STOPPED = 0,
0033     /* AUX transaction is on, BTS tracing is disabled */
0034     BTS_STATE_INACTIVE,
0035     /* AUX transaction is on, BTS tracing is running */
0036     BTS_STATE_ACTIVE,
0037 };
0038 
0039 static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
0040 
0041 #define BTS_RECORD_SIZE     24
0042 #define BTS_SAFETY_MARGIN   4080
0043 
0044 struct bts_phys {
0045     struct page *page;
0046     unsigned long   size;
0047     unsigned long   offset;
0048     unsigned long   displacement;
0049 };
0050 
0051 struct bts_buffer {
0052     size_t      real_size;  /* multiple of BTS_RECORD_SIZE */
0053     unsigned int    nr_pages;
0054     unsigned int    nr_bufs;
0055     unsigned int    cur_buf;
0056     bool        snapshot;
0057     local_t     data_size;
0058     local_t     head;
0059     unsigned long   end;
0060     void        **data_pages;
0061     struct bts_phys buf[];
0062 };
0063 
0064 static struct pmu bts_pmu;
0065 
0066 static int buf_nr_pages(struct page *page)
0067 {
0068     if (!PagePrivate(page))
0069         return 1;
0070 
0071     return 1 << page_private(page);
0072 }
0073 
0074 static size_t buf_size(struct page *page)
0075 {
0076     return buf_nr_pages(page) * PAGE_SIZE;
0077 }
0078 
0079 static void *
0080 bts_buffer_setup_aux(struct perf_event *event, void **pages,
0081              int nr_pages, bool overwrite)
0082 {
0083     struct bts_buffer *buf;
0084     struct page *page;
0085     int cpu = event->cpu;
0086     int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
0087     unsigned long offset;
0088     size_t size = nr_pages << PAGE_SHIFT;
0089     int pg, nbuf, pad;
0090 
0091     /* count all the high order buffers */
0092     for (pg = 0, nbuf = 0; pg < nr_pages;) {
0093         page = virt_to_page(pages[pg]);
0094         pg += buf_nr_pages(page);
0095         nbuf++;
0096     }
0097 
0098     /*
0099      * to avoid interrupts in overwrite mode, only allow one physical
0100      */
0101     if (overwrite && nbuf > 1)
0102         return NULL;
0103 
0104     buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
0105     if (!buf)
0106         return NULL;
0107 
0108     buf->nr_pages = nr_pages;
0109     buf->nr_bufs = nbuf;
0110     buf->snapshot = overwrite;
0111     buf->data_pages = pages;
0112     buf->real_size = size - size % BTS_RECORD_SIZE;
0113 
0114     for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
0115         unsigned int __nr_pages;
0116 
0117         page = virt_to_page(pages[pg]);
0118         __nr_pages = buf_nr_pages(page);
0119         buf->buf[nbuf].page = page;
0120         buf->buf[nbuf].offset = offset;
0121         buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
0122         buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
0123         pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
0124         buf->buf[nbuf].size -= pad;
0125 
0126         pg += __nr_pages;
0127         offset += __nr_pages << PAGE_SHIFT;
0128     }
0129 
0130     return buf;
0131 }
0132 
0133 static void bts_buffer_free_aux(void *data)
0134 {
0135     kfree(data);
0136 }
0137 
0138 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
0139 {
0140     return buf->buf[idx].offset + buf->buf[idx].displacement;
0141 }
0142 
0143 static void
0144 bts_config_buffer(struct bts_buffer *buf)
0145 {
0146     int cpu = raw_smp_processor_id();
0147     struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
0148     struct bts_phys *phys = &buf->buf[buf->cur_buf];
0149     unsigned long index, thresh = 0, end = phys->size;
0150     struct page *page = phys->page;
0151 
0152     index = local_read(&buf->head);
0153 
0154     if (!buf->snapshot) {
0155         if (buf->end < phys->offset + buf_size(page))
0156             end = buf->end - phys->offset - phys->displacement;
0157 
0158         index -= phys->offset + phys->displacement;
0159 
0160         if (end - index > BTS_SAFETY_MARGIN)
0161             thresh = end - BTS_SAFETY_MARGIN;
0162         else if (end - index > BTS_RECORD_SIZE)
0163             thresh = end - BTS_RECORD_SIZE;
0164         else
0165             thresh = end;
0166     }
0167 
0168     ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
0169     ds->bts_index = ds->bts_buffer_base + index;
0170     ds->bts_absolute_maximum = ds->bts_buffer_base + end;
0171     ds->bts_interrupt_threshold = !buf->snapshot
0172         ? ds->bts_buffer_base + thresh
0173         : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
0174 }
0175 
0176 static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
0177 {
0178     unsigned long index = head - phys->offset;
0179 
0180     memset(page_address(phys->page) + index, 0, phys->size - index);
0181 }
0182 
0183 static void bts_update(struct bts_ctx *bts)
0184 {
0185     int cpu = raw_smp_processor_id();
0186     struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
0187     struct bts_buffer *buf = perf_get_aux(&bts->handle);
0188     unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
0189 
0190     if (!buf)
0191         return;
0192 
0193     head = index + bts_buffer_offset(buf, buf->cur_buf);
0194     old = local_xchg(&buf->head, head);
0195 
0196     if (!buf->snapshot) {
0197         if (old == head)
0198             return;
0199 
0200         if (ds->bts_index >= ds->bts_absolute_maximum)
0201             perf_aux_output_flag(&bts->handle,
0202                                  PERF_AUX_FLAG_TRUNCATED);
0203 
0204         /*
0205          * old and head are always in the same physical buffer, so we
0206          * can subtract them to get the data size.
0207          */
0208         local_add(head - old, &buf->data_size);
0209     } else {
0210         local_set(&buf->data_size, head);
0211     }
0212 
0213     /*
0214      * Since BTS is coherent, just add compiler barrier to ensure
0215      * BTS updating is ordered against bts::handle::event.
0216      */
0217     barrier();
0218 }
0219 
0220 static int
0221 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
0222 
0223 /*
0224  * Ordering PMU callbacks wrt themselves and the PMI is done by means
0225  * of bts::state, which:
0226  *  - is set when bts::handle::event is valid, that is, between
0227  *    perf_aux_output_begin() and perf_aux_output_end();
0228  *  - is zero otherwise;
0229  *  - is ordered against bts::handle::event with a compiler barrier.
0230  */
0231 
0232 static void __bts_event_start(struct perf_event *event)
0233 {
0234     struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
0235     struct bts_buffer *buf = perf_get_aux(&bts->handle);
0236     u64 config = 0;
0237 
0238     if (!buf->snapshot)
0239         config |= ARCH_PERFMON_EVENTSEL_INT;
0240     if (!event->attr.exclude_kernel)
0241         config |= ARCH_PERFMON_EVENTSEL_OS;
0242     if (!event->attr.exclude_user)
0243         config |= ARCH_PERFMON_EVENTSEL_USR;
0244 
0245     bts_config_buffer(buf);
0246 
0247     /*
0248      * local barrier to make sure that ds configuration made it
0249      * before we enable BTS and bts::state goes ACTIVE
0250      */
0251     wmb();
0252 
0253     /* INACTIVE/STOPPED -> ACTIVE */
0254     WRITE_ONCE(bts->state, BTS_STATE_ACTIVE);
0255 
0256     intel_pmu_enable_bts(config);
0257 
0258 }
0259 
0260 static void bts_event_start(struct perf_event *event, int flags)
0261 {
0262     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0263     struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
0264     struct bts_buffer *buf;
0265 
0266     buf = perf_aux_output_begin(&bts->handle, event);
0267     if (!buf)
0268         goto fail_stop;
0269 
0270     if (bts_buffer_reset(buf, &bts->handle))
0271         goto fail_end_stop;
0272 
0273     bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
0274     bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
0275     bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
0276 
0277     perf_event_itrace_started(event);
0278     event->hw.state = 0;
0279 
0280     __bts_event_start(event);
0281 
0282     return;
0283 
0284 fail_end_stop:
0285     perf_aux_output_end(&bts->handle, 0);
0286 
0287 fail_stop:
0288     event->hw.state = PERF_HES_STOPPED;
0289 }
0290 
0291 static void __bts_event_stop(struct perf_event *event, int state)
0292 {
0293     struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
0294 
0295     /* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */
0296     WRITE_ONCE(bts->state, state);
0297 
0298     /*
0299      * No extra synchronization is mandated by the documentation to have
0300      * BTS data stores globally visible.
0301      */
0302     intel_pmu_disable_bts();
0303 }
0304 
0305 static void bts_event_stop(struct perf_event *event, int flags)
0306 {
0307     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0308     struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
0309     struct bts_buffer *buf = NULL;
0310     int state = READ_ONCE(bts->state);
0311 
0312     if (state == BTS_STATE_ACTIVE)
0313         __bts_event_stop(event, BTS_STATE_STOPPED);
0314 
0315     if (state != BTS_STATE_STOPPED)
0316         buf = perf_get_aux(&bts->handle);
0317 
0318     event->hw.state |= PERF_HES_STOPPED;
0319 
0320     if (flags & PERF_EF_UPDATE) {
0321         bts_update(bts);
0322 
0323         if (buf) {
0324             if (buf->snapshot)
0325                 bts->handle.head =
0326                     local_xchg(&buf->data_size,
0327                            buf->nr_pages << PAGE_SHIFT);
0328             perf_aux_output_end(&bts->handle,
0329                                 local_xchg(&buf->data_size, 0));
0330         }
0331 
0332         cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
0333         cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
0334         cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
0335         cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
0336     }
0337 }
0338 
0339 void intel_bts_enable_local(void)
0340 {
0341     struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
0342     int state = READ_ONCE(bts->state);
0343 
0344     /*
0345      * Here we transition from INACTIVE to ACTIVE;
0346      * if we instead are STOPPED from the interrupt handler,
0347      * stay that way. Can't be ACTIVE here though.
0348      */
0349     if (WARN_ON_ONCE(state == BTS_STATE_ACTIVE))
0350         return;
0351 
0352     if (state == BTS_STATE_STOPPED)
0353         return;
0354 
0355     if (bts->handle.event)
0356         __bts_event_start(bts->handle.event);
0357 }
0358 
0359 void intel_bts_disable_local(void)
0360 {
0361     struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
0362 
0363     /*
0364      * Here we transition from ACTIVE to INACTIVE;
0365      * do nothing for STOPPED or INACTIVE.
0366      */
0367     if (READ_ONCE(bts->state) != BTS_STATE_ACTIVE)
0368         return;
0369 
0370     if (bts->handle.event)
0371         __bts_event_stop(bts->handle.event, BTS_STATE_INACTIVE);
0372 }
0373 
0374 static int
0375 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
0376 {
0377     unsigned long head, space, next_space, pad, gap, skip, wakeup;
0378     unsigned int next_buf;
0379     struct bts_phys *phys, *next_phys;
0380     int ret;
0381 
0382     if (buf->snapshot)
0383         return 0;
0384 
0385     head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
0386 
0387     phys = &buf->buf[buf->cur_buf];
0388     space = phys->offset + phys->displacement + phys->size - head;
0389     pad = space;
0390     if (space > handle->size) {
0391         space = handle->size;
0392         space -= space % BTS_RECORD_SIZE;
0393     }
0394     if (space <= BTS_SAFETY_MARGIN) {
0395         /* See if next phys buffer has more space */
0396         next_buf = buf->cur_buf + 1;
0397         if (next_buf >= buf->nr_bufs)
0398             next_buf = 0;
0399         next_phys = &buf->buf[next_buf];
0400         gap = buf_size(phys->page) - phys->displacement - phys->size +
0401               next_phys->displacement;
0402         skip = pad + gap;
0403         if (handle->size >= skip) {
0404             next_space = next_phys->size;
0405             if (next_space + skip > handle->size) {
0406                 next_space = handle->size - skip;
0407                 next_space -= next_space % BTS_RECORD_SIZE;
0408             }
0409             if (next_space > space || !space) {
0410                 if (pad)
0411                     bts_buffer_pad_out(phys, head);
0412                 ret = perf_aux_output_skip(handle, skip);
0413                 if (ret)
0414                     return ret;
0415                 /* Advance to next phys buffer */
0416                 phys = next_phys;
0417                 space = next_space;
0418                 head = phys->offset + phys->displacement;
0419                 /*
0420                  * After this, cur_buf and head won't match ds
0421                  * anymore, so we must not be racing with
0422                  * bts_update().
0423                  */
0424                 buf->cur_buf = next_buf;
0425                 local_set(&buf->head, head);
0426             }
0427         }
0428     }
0429 
0430     /* Don't go far beyond wakeup watermark */
0431     wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
0432          handle->head;
0433     if (space > wakeup) {
0434         space = wakeup;
0435         space -= space % BTS_RECORD_SIZE;
0436     }
0437 
0438     buf->end = head + space;
0439 
0440     /*
0441      * If we have no space, the lost notification would have been sent when
0442      * we hit absolute_maximum - see bts_update()
0443      */
0444     if (!space)
0445         return -ENOSPC;
0446 
0447     return 0;
0448 }
0449 
0450 int intel_bts_interrupt(void)
0451 {
0452     struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
0453     struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
0454     struct perf_event *event = bts->handle.event;
0455     struct bts_buffer *buf;
0456     s64 old_head;
0457     int err = -ENOSPC, handled = 0;
0458 
0459     /*
0460      * The only surefire way of knowing if this NMI is ours is by checking
0461      * the write ptr against the PMI threshold.
0462      */
0463     if (ds && (ds->bts_index >= ds->bts_interrupt_threshold))
0464         handled = 1;
0465 
0466     /*
0467      * this is wrapped in intel_bts_enable_local/intel_bts_disable_local,
0468      * so we can only be INACTIVE or STOPPED
0469      */
0470     if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
0471         return handled;
0472 
0473     buf = perf_get_aux(&bts->handle);
0474     if (!buf)
0475         return handled;
0476 
0477     /*
0478      * Skip snapshot counters: they don't use the interrupt, but
0479      * there's no other way of telling, because the pointer will
0480      * keep moving
0481      */
0482     if (buf->snapshot)
0483         return 0;
0484 
0485     old_head = local_read(&buf->head);
0486     bts_update(bts);
0487 
0488     /* no new data */
0489     if (old_head == local_read(&buf->head))
0490         return handled;
0491 
0492     perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
0493 
0494     buf = perf_aux_output_begin(&bts->handle, event);
0495     if (buf)
0496         err = bts_buffer_reset(buf, &bts->handle);
0497 
0498     if (err) {
0499         WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
0500 
0501         if (buf) {
0502             /*
0503              * BTS_STATE_STOPPED should be visible before
0504              * cleared handle::event
0505              */
0506             barrier();
0507             perf_aux_output_end(&bts->handle, 0);
0508         }
0509     }
0510 
0511     return 1;
0512 }
0513 
0514 static void bts_event_del(struct perf_event *event, int mode)
0515 {
0516     bts_event_stop(event, PERF_EF_UPDATE);
0517 }
0518 
0519 static int bts_event_add(struct perf_event *event, int mode)
0520 {
0521     struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
0522     struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
0523     struct hw_perf_event *hwc = &event->hw;
0524 
0525     event->hw.state = PERF_HES_STOPPED;
0526 
0527     if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
0528         return -EBUSY;
0529 
0530     if (bts->handle.event)
0531         return -EBUSY;
0532 
0533     if (mode & PERF_EF_START) {
0534         bts_event_start(event, 0);
0535         if (hwc->state & PERF_HES_STOPPED)
0536             return -EINVAL;
0537     }
0538 
0539     return 0;
0540 }
0541 
0542 static void bts_event_destroy(struct perf_event *event)
0543 {
0544     x86_release_hardware();
0545     x86_del_exclusive(x86_lbr_exclusive_bts);
0546 }
0547 
0548 static int bts_event_init(struct perf_event *event)
0549 {
0550     int ret;
0551 
0552     if (event->attr.type != bts_pmu.type)
0553         return -ENOENT;
0554 
0555     /*
0556      * BTS leaks kernel addresses even when CPL0 tracing is
0557      * disabled, so disallow intel_bts driver for unprivileged
0558      * users on paranoid systems since it provides trace data
0559      * to the user in a zero-copy fashion.
0560      *
0561      * Note that the default paranoia setting permits unprivileged
0562      * users to profile the kernel.
0563      */
0564     if (event->attr.exclude_kernel) {
0565         ret = perf_allow_kernel(&event->attr);
0566         if (ret)
0567             return ret;
0568     }
0569 
0570     if (x86_add_exclusive(x86_lbr_exclusive_bts))
0571         return -EBUSY;
0572 
0573     ret = x86_reserve_hardware();
0574     if (ret) {
0575         x86_del_exclusive(x86_lbr_exclusive_bts);
0576         return ret;
0577     }
0578 
0579     event->destroy = bts_event_destroy;
0580 
0581     return 0;
0582 }
0583 
0584 static void bts_event_read(struct perf_event *event)
0585 {
0586 }
0587 
0588 static __init int bts_init(void)
0589 {
0590     if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
0591         return -ENODEV;
0592 
0593     if (boot_cpu_has(X86_FEATURE_PTI)) {
0594         /*
0595          * BTS hardware writes through a virtual memory map we must
0596          * either use the kernel physical map, or the user mapping of
0597          * the AUX buffer.
0598          *
0599          * However, since this driver supports per-CPU and per-task inherit
0600          * we cannot use the user mapping since it will not be available
0601          * if we're not running the owning process.
0602          *
0603          * With PTI we can't use the kernel map either, because its not
0604          * there when we run userspace.
0605          *
0606          * For now, disable this driver when using PTI.
0607          */
0608         return -ENODEV;
0609     }
0610 
0611     bts_pmu.capabilities    = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE |
0612                   PERF_PMU_CAP_EXCLUSIVE;
0613     bts_pmu.task_ctx_nr = perf_sw_context;
0614     bts_pmu.event_init  = bts_event_init;
0615     bts_pmu.add     = bts_event_add;
0616     bts_pmu.del     = bts_event_del;
0617     bts_pmu.start       = bts_event_start;
0618     bts_pmu.stop        = bts_event_stop;
0619     bts_pmu.read        = bts_event_read;
0620     bts_pmu.setup_aux   = bts_buffer_setup_aux;
0621     bts_pmu.free_aux    = bts_buffer_free_aux;
0622 
0623     return perf_pmu_register(&bts_pmu, "intel_bts", -1);
0624 }
0625 arch_initcall(bts_init);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!