OSCL-LXR linux-6.0.1/​drivers/​hwtracing/​coresight/​coresight-trbe.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0
 /*
  * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight
  * sink device could then pair with an appropriate per-cpu coresight source
  * device (ETE) thus generating required trace data. Trace can be enabled
  * via the perf framework.
  *
  * The AUX buffer handling is inspired from Arm SPE PMU driver.
  *
  * Copyright (C) 2020 ARM Ltd.
  *
  * Author: Anshuman Khandual <anshuman.khandual@arm.com>
  */
 #define DRVNAME "arm_trbe"
 
 #define pr_fmt(fmt) DRVNAME ": " fmt
 
 #include <asm/barrier.h>
 #include <asm/cpufeature.h>
 
 #include "coresight-self-hosted-trace.h"
 #include "coresight-trbe.h"
 
 #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
 
 /*
  * A padding packet that will help the user space tools
  * in skipping relevant sections in the captured trace
  * data which could not be decoded. TRBE doesn't support
  * formatting the trace data, unlike the legacy CoreSight
  * sinks and thus we use ETE trace packets to pad the
  * sections of the buffer.
  */
 #define ETE_IGNORE_PACKET       0x70
 
 /*
  * Minimum amount of meaningful trace will contain:
  * A-Sync, Trace Info, Trace On, Address, Atom.
  * This is about 44bytes of ETE trace. To be on
  * the safer side, we assume 64bytes is the minimum
  * space required for a meaningful session, before
  * we hit a "WRAP" event.
  */
 #define TRBE_TRACE_MIN_BUF_SIZE     64
 
 enum trbe_fault_action {
     TRBE_FAULT_ACT_WRAP,
     TRBE_FAULT_ACT_SPURIOUS,
     TRBE_FAULT_ACT_FATAL,
 };
 
 struct trbe_buf {
     /*
      * Even though trbe_base represents vmap()
      * mapped allocated buffer's start address,
      * it's being as unsigned long for various
      * arithmetic and comparision operations &
      * also to be consistent with trbe_write &
      * trbe_limit sibling pointers.
      */
     unsigned long trbe_base;
     /* The base programmed into the TRBE */
     unsigned long trbe_hw_base;
     unsigned long trbe_limit;
     unsigned long trbe_write;
     int nr_pages;
     void **pages;
     bool snapshot;
     struct trbe_cpudata *cpudata;
 };
 
 /*
  * TRBE erratum list
  *
  * The errata are defined in arm64 generic cpu_errata framework.
  * Since the errata work arounds could be applied individually
  * to the affected CPUs inside the TRBE driver, we need to know if
  * a given CPU is affected by the erratum. Unlike the other erratum
  * work arounds, TRBE driver needs to check multiple times during
  * a trace session. Thus we need a quicker access to per-CPU
  * errata and not issue costly this_cpu_has_cap() everytime.
  * We keep a set of the affected errata in trbe_cpudata, per TRBE.
  *
  * We rely on the corresponding cpucaps to be defined for a given
  * TRBE erratum. We map the given cpucap into a TRBE internal number
  * to make the tracking of the errata lean.
  *
  * This helps in :
  *   - Not duplicating the detection logic
  *   - Streamlined detection of erratum across the system
  */
 #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE 0
 #define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE  1
 #define TRBE_NEEDS_DRAIN_AFTER_DISABLE      2
 #define TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE   3
 #define TRBE_IS_BROKEN              4
 
 static int trbe_errata_cpucaps[] = {
     [TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,
     [TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,
     [TRBE_NEEDS_DRAIN_AFTER_DISABLE] = ARM64_WORKAROUND_2064142,
     [TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE] = ARM64_WORKAROUND_2038923,
     [TRBE_IS_BROKEN] = ARM64_WORKAROUND_1902691,
     -1,     /* Sentinel, must be the last entry */
 };
 
 /* The total number of listed errata in trbe_errata_cpucaps */
 #define TRBE_ERRATA_MAX         (ARRAY_SIZE(trbe_errata_cpucaps) - 1)
 
 /*
  * Safe limit for the number of bytes that may be overwritten
  * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered.
  */
 #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES  256
 
 /*
  * struct trbe_cpudata: TRBE instance specific data
  * @trbe_flag       - TRBE dirty/access flag support
  * @trbe_hw_align   - Actual TRBE alignment required for TRBPTR_EL1.
  * @trbe_align      - Software alignment used for the TRBPTR_EL1.
  * @cpu         - CPU this TRBE belongs to.
  * @mode        - Mode of current operation. (perf/disabled)
  * @drvdata     - TRBE specific drvdata
  * @errata      - Bit map for the errata on this TRBE.
  */
 struct trbe_cpudata {
     bool trbe_flag;
     u64 trbe_hw_align;
     u64 trbe_align;
     int cpu;
     enum cs_mode mode;
     struct trbe_buf *buf;
     struct trbe_drvdata *drvdata;
     DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
 };
 
 struct trbe_drvdata {
     struct trbe_cpudata __percpu *cpudata;
     struct perf_output_handle * __percpu *handle;
     struct hlist_node hotplug_node;
     int irq;
     cpumask_t supported_cpus;
     enum cpuhp_state trbe_online;
     struct platform_device *pdev;
 };
 
 static void trbe_check_errata(struct trbe_cpudata *cpudata)
 {
     int i;
 
     for (i = 0; i < TRBE_ERRATA_MAX; i++) {
         int cap = trbe_errata_cpucaps[i];
 
         if (WARN_ON_ONCE(cap < 0))
             return;
         if (this_cpu_has_cap(cap))
             set_bit(i, cpudata->errata);
     }
 }
 
 static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)
 {
     return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);
 }
 
 static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)
 {
     return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);
 }
 
 static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)
 {
     return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);
 }
 
 static inline bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)
 {
     /*
      * Errata affected TRBE implementation will need TSB CSYNC and
      * DSB in order to prevent subsequent writes into certain TRBE
      * system registers from being ignored and not effected.
      */
     return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);
 }
 
 static inline bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)
 {
     /*
      * Errata affected TRBE implementation will need an additional
      * context synchronization in order to prevent an inconsistent
      * TRBE prohibited region view on the CPU which could possibly
      * corrupt the TRBE buffer or the TRBE state.
      */
     return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);
 }
 
 static inline bool trbe_is_broken(struct trbe_cpudata *cpudata)
 {
     return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);
 }
 
 static int trbe_alloc_node(struct perf_event *event)
 {
     if (event->cpu == -1)
         return NUMA_NO_NODE;
     return cpu_to_node(event->cpu);
 }
 
 static inline void trbe_drain_buffer(void)
 {
     tsb_csync();
     dsb(nsh);
 }
 
 static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)
 {
     /*
      * Enable the TRBE without clearing LIMITPTR which
      * might be required for fetching the buffer limits.
      */
     trblimitr |= TRBLIMITR_ENABLE;
     write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
 
     /* Synchronize the TRBE enable event */
     isb();
 
     if (trbe_needs_ctxt_sync_after_enable(cpudata))
         isb();
 }
 
 static inline void set_trbe_disabled(struct trbe_cpudata *cpudata)
 {
     u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
 
     /*
      * Disable the TRBE without clearing LIMITPTR which
      * might be required for fetching the buffer limits.
      */
     trblimitr &= ~TRBLIMITR_ENABLE;
     write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
 
     if (trbe_needs_drain_after_disable(cpudata))
         trbe_drain_buffer();
     isb();
 }
 
 static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)
 {
     trbe_drain_buffer();
     set_trbe_disabled(cpudata);
 }
 
 static void trbe_reset_local(struct trbe_cpudata *cpudata)
 {
     trbe_drain_and_disable_local(cpudata);
     write_sysreg_s(0, SYS_TRBLIMITR_EL1);
     write_sysreg_s(0, SYS_TRBPTR_EL1);
     write_sysreg_s(0, SYS_TRBBASER_EL1);
     write_sysreg_s(0, SYS_TRBSR_EL1);
 }
 
 static void trbe_report_wrap_event(struct perf_output_handle *handle)
 {
     /*
      * Mark the buffer to indicate that there was a WRAP event by
      * setting the COLLISION flag. This indicates to the user that
      * the TRBE trace collection was stopped without stopping the
      * ETE and thus there might be some amount of trace that was
      * lost between the time the WRAP was detected and the IRQ
      * was consumed by the CPU.
      *
      * Setting the TRUNCATED flag would move the event to STOPPED
      * state unnecessarily, even when there is space left in the
      * ring buffer. Using the COLLISION flag doesn't have this side
      * effect. We only set TRUNCATED flag when there is no space
      * left in the ring buffer.
      */
     perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
 }
 
 static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)
 {
     struct trbe_buf *buf = etm_perf_sink_config(handle);
 
     /*
      * We cannot proceed with the buffer collection and we
      * do not have any data for the current session. The
      * etm_perf driver expects to close out the aux_buffer
      * at event_stop(). So disable the TRBE here and leave
      * the update_buffer() to return a 0 size.
      */
     trbe_drain_and_disable_local(buf->cpudata);
     perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
     perf_aux_output_end(handle, 0);
     *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
 }
 
 /*
  * TRBE Buffer Management
  *
  * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,
  * it starts writing trace data from the write pointer onward till the limit pointer.
  * When the write pointer reaches the address just before the limit pointer, it gets
  * wrapped around again to the base pointer. This is called a TRBE wrap event, which
  * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver
  * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ
  * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and
  * LIMIT pointers.
  *
  *  Wrap around with an IRQ
  *  ------ < ------ < ------- < ----- < -----
  *  |                   |
  *  ------ > ------ > ------- > ----- > -----
  *
  *  +---------------+-----------------------+
  *  |       |           |
  *  +---------------+-----------------------+
  *  Base Pointer    Write Pointer       Limit Pointer
  *
  * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write
  * pointer can be aligned to the implementation defined TRBE trace buffer alignment
  * as captured in trbe_cpudata->trbe_align.
  *
  *
  *      head        tail        wakeup
  *  +---------------------------------------+----- ~ ~ ------
  *  |$$$$$$$|################|$$$$$$$$$$$$$$|       |
  *  +---------------------------------------+----- ~ ~ ------
  *  Base Pointer    Write Pointer       Limit Pointer
  *
  * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing
  * values which tracks all the driver writes and user reads from the perf auxiliary
  * buffer. Generally [head..tail] is the area where the driver can write into unless
  * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and
  * configured depending on the perf_output_handle indices, so that the driver does
  * not override into areas in the perf auxiliary buffer which is being or yet to be
  * consumed from the user space. The enabled TRBE buffer area is a moving subset of
  * the allocated perf auxiliary buffer.
  */
 
 static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)
 {
     memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);
 }
 
 static void trbe_pad_buf(struct perf_output_handle *handle, int len)
 {
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     u64 head = PERF_IDX2OFF(handle->head, buf);
 
     __trbe_pad_buf(buf, head, len);
     if (!buf->snapshot)
         perf_aux_output_skip(handle, len);
 }
 
 static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)
 {
     struct trbe_buf *buf = etm_perf_sink_config(handle);
 
     /*
      * The ETE trace has alignment synchronization packets allowing
      * the decoder to reset in case of an overflow or corruption.
      * So we can use the entire buffer for the snapshot mode.
      */
     return buf->nr_pages * PAGE_SIZE;
 }
 
 static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)
 {
     u64 size = TRBE_TRACE_MIN_BUF_SIZE;
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     struct trbe_cpudata *cpudata = buf->cpudata;
 
     /*
      * When the TRBE is affected by an erratum that could make it
      * write to the next "virtually addressed" page beyond the LIMIT.
      * We need to make sure there is always a PAGE after the LIMIT,
      * within the buffer. Thus we ensure there is at least an extra
      * page than normal. With this we could then adjust the LIMIT
      * pointer down by a PAGE later.
      */
     if (trbe_may_write_out_of_range(cpudata))
         size += PAGE_SIZE;
     return size;
 }
 
 /*
  * TRBE Limit Calculation
  *
  * The following markers are used to illustrate various TRBE buffer situations.
  *
  * $$$$ - Data area, unconsumed captured trace data, not to be overridden
  * #### - Free area, enabled, trace will be written
  * %%%% - Free area, disabled, trace will not be written
  * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped
  */
 static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
 {
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     struct trbe_cpudata *cpudata = buf->cpudata;
     const u64 bufsize = buf->nr_pages * PAGE_SIZE;
     u64 limit = bufsize;
     u64 head, tail, wakeup;
 
     head = PERF_IDX2OFF(handle->head, buf);
 
     /*
      *      head
      *  ------->|
      *  |
      *  head    TRBE align  tail
      * +----|-------|---------------|-------+
      * |$$$$|=======|###############|$$$$$$$|
      * +----|-------|---------------|-------+
      * trbe_base                trbe_base + nr_pages
      *
      * Perf aux buffer output head position can be misaligned depending on
      * various factors including user space reads. In case misaligned, head
      * needs to be aligned before TRBE can be configured. Pad the alignment
      * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools
      * and skip this section thus advancing the head.
      */
     if (!IS_ALIGNED(head, cpudata->trbe_align)) {
         unsigned long delta = roundup(head, cpudata->trbe_align) - head;
 
         delta = min(delta, handle->size);
         trbe_pad_buf(handle, delta);
         head = PERF_IDX2OFF(handle->head, buf);
     }
 
     /*
      *  head = tail (size = 0)
      * +----|-------------------------------+
      * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ |
      * +----|-------------------------------+
      * trbe_base                trbe_base + nr_pages
      *
      * Perf aux buffer does not have any space for the driver to write into.
      */
     if (!handle->size)
         return 0;
 
     /* Compute the tail and wakeup indices now that we've aligned head */
     tail = PERF_IDX2OFF(handle->head + handle->size, buf);
     wakeup = PERF_IDX2OFF(handle->wakeup, buf);
 
     /*
      * Lets calculate the buffer area which TRBE could write into. There
      * are three possible scenarios here. Limit needs to be aligned with
      * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the
      * unconsumed data.
      *
      * 1) head < tail
      *
      *  head            tail
      * +----|-----------------------|-------+
      * |$$$$|#######################|$$$$$$$|
      * +----|-----------------------|-------+
      * trbe_base            limit   trbe_base + nr_pages
      *
      * TRBE could write into [head..tail] area. Unless the tail is right at
      * the end of the buffer, neither an wrap around nor an IRQ is expected
      * while being enabled.
      *
      * 2) head == tail
      *
      *  head = tail (size > 0)
      * +----|-------------------------------+
      * |%%%%|###############################|
      * +----|-------------------------------+
      * trbe_base                limit = trbe_base + nr_pages
      *
      * TRBE should just write into [head..base + nr_pages] area even though
      * the entire buffer is empty. Reason being, when the trace reaches the
      * end of the buffer, it will just wrap around with an IRQ giving an
      * opportunity to reconfigure the buffer.
      *
      * 3) tail < head
      *
      *  tail            head
      * +----|-----------------------|-------+
      * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|
      * +----|-----------------------|-------+
      * trbe_base                limit = trbe_base + nr_pages
      *
      * TRBE should just write into [head..base + nr_pages] area even though
      * the [trbe_base..tail] is also empty. Reason being, when the trace
      * reaches the end of the buffer, it will just wrap around with an IRQ
      * giving an opportunity to reconfigure the buffer.
      */
     if (head < tail)
         limit = round_down(tail, PAGE_SIZE);
 
     /*
      * Wakeup may be arbitrarily far into the future. If it's not in the
      * current generation, either we'll wrap before hitting it, or it's
      * in the past and has been handled already.
      *
      * If there's a wakeup before we wrap, arrange to be woken up by the
      * page boundary following it. Keep the tail boundary if that's lower.
      *
      *  head        wakeup  tail
      * +----|---------------|-------|-------+
      * |$$$$|###############|%%%%%%%|$$$$$$$|
      * +----|---------------|-------|-------+
      * trbe_base        limit       trbe_base + nr_pages
      */
     if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
         limit = min(limit, round_up(wakeup, PAGE_SIZE));
 
     /*
      * There are two situation when this can happen i.e limit is before
      * the head and hence TRBE cannot be configured.
      *
      * 1) head < tail (aligned down with PAGE_SIZE) and also they are both
      * within the same PAGE size range.
      *
      *          PAGE_SIZE
      *      |----------------------|
      *
      *      limit   head    tail
      * +------------|------|--------|-------+
      * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|
      * +------------|------|--------|-------+
      * trbe_base                trbe_base + nr_pages
      *
      * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both
      * head and wakeup are within same PAGE size range.
      *
      *      PAGE_SIZE
      *  |----------------------|
      *
      *  limit   head    wakeup  tail
      * +----|------|-------|--------|-------+
      * |$$$$$$$$$$$|=======|========|$$$$$$$|
      * +----|------|-------|--------|-------+
      * trbe_base                trbe_base + nr_pages
      */
     if (limit > head)
         return limit;
 
     trbe_pad_buf(handle, handle->size);
     return 0;
 }
 
 static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
 {
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     u64 limit = __trbe_normal_offset(handle);
     u64 head = PERF_IDX2OFF(handle->head, buf);
 
     /*
      * If the head is too close to the limit and we don't
      * have space for a meaningful run, we rather pad it
      * and start fresh.
      *
      * We might have to do this more than once to make sure
      * we have enough required space.
      */
     while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {
         trbe_pad_buf(handle, limit - head);
         limit = __trbe_normal_offset(handle);
         head = PERF_IDX2OFF(handle->head, buf);
     }
     return limit;
 }
 
 static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)
 {
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     unsigned long offset;
 
     if (buf->snapshot)
         offset = trbe_snapshot_offset(handle);
     else
         offset = trbe_normal_offset(handle);
     return buf->trbe_base + offset;
 }
 
 static void clr_trbe_status(void)
 {
     u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);
 
     WARN_ON(is_trbe_enabled());
     trbsr &= ~TRBSR_IRQ;
     trbsr &= ~TRBSR_TRG;
     trbsr &= ~TRBSR_WRAP;
     trbsr &= ~(TRBSR_EC_MASK << TRBSR_EC_SHIFT);
     trbsr &= ~(TRBSR_BSC_MASK << TRBSR_BSC_SHIFT);
     trbsr &= ~TRBSR_STOP;
     write_sysreg_s(trbsr, SYS_TRBSR_EL1);
 }
 
 static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)
 {
     u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
     unsigned long addr = buf->trbe_limit;
 
     WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_LIMIT_SHIFT)));
     WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));
 
     trblimitr &= ~TRBLIMITR_NVM;
     trblimitr &= ~(TRBLIMITR_FILL_MODE_MASK << TRBLIMITR_FILL_MODE_SHIFT);
     trblimitr &= ~(TRBLIMITR_TRIG_MODE_MASK << TRBLIMITR_TRIG_MODE_SHIFT);
     trblimitr &= ~(TRBLIMITR_LIMIT_MASK << TRBLIMITR_LIMIT_SHIFT);
 
     /*
      * Fill trace buffer mode is used here while configuring the
      * TRBE for trace capture. In this particular mode, the trace
      * collection is stopped and a maintenance interrupt is raised
      * when the current write pointer wraps. This pause in trace
      * collection gives the software an opportunity to capture the
      * trace data in the interrupt handler, before reconfiguring
      * the TRBE.
      */
     trblimitr |= (TRBE_FILL_MODE_FILL & TRBLIMITR_FILL_MODE_MASK) << TRBLIMITR_FILL_MODE_SHIFT;
 
     /*
      * Trigger mode is not used here while configuring the TRBE for
      * the trace capture. Hence just keep this in the ignore mode.
      */
     trblimitr |= (TRBE_TRIG_MODE_IGNORE & TRBLIMITR_TRIG_MODE_MASK) <<
               TRBLIMITR_TRIG_MODE_SHIFT;
     trblimitr |= (addr & PAGE_MASK);
     set_trbe_enabled(buf->cpudata, trblimitr);
 }
 
 static void trbe_enable_hw(struct trbe_buf *buf)
 {
     WARN_ON(buf->trbe_hw_base < buf->trbe_base);
     WARN_ON(buf->trbe_write < buf->trbe_hw_base);
     WARN_ON(buf->trbe_write >= buf->trbe_limit);
     set_trbe_disabled(buf->cpudata);
     clr_trbe_status();
     set_trbe_base_pointer(buf->trbe_hw_base);
     set_trbe_write_pointer(buf->trbe_write);
 
     /*
      * Synchronize all the register updates
      * till now before enabling the TRBE.
      */
     isb();
     set_trbe_limit_pointer_enabled(buf);
 }
 
 static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,
                          u64 trbsr)
 {
     int ec = get_trbe_ec(trbsr);
     int bsc = get_trbe_bsc(trbsr);
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     struct trbe_cpudata *cpudata = buf->cpudata;
 
     WARN_ON(is_trbe_running(trbsr));
     if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))
         return TRBE_FAULT_ACT_FATAL;
 
     if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))
         return TRBE_FAULT_ACT_FATAL;
 
     /*
      * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
      * it might write data after a WRAP event in the fill mode.
      * Thus the check TRBPTR == TRBBASER will not be honored.
      */
     if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) &&
         (trbe_may_overwrite_in_fill_mode(cpudata) ||
          get_trbe_write_pointer() == get_trbe_base_pointer()))
         return TRBE_FAULT_ACT_WRAP;
 
     return TRBE_FAULT_ACT_SPURIOUS;
 }
 
 static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,
                      struct trbe_buf *buf, bool wrap)
 {
     u64 write;
     u64 start_off, end_off;
     u64 size;
     u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
 
     /*
      * If the TRBE has wrapped around the write pointer has
      * wrapped and should be treated as limit.
      *
      * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE,
      * it may write upto 64bytes beyond the "LIMIT". The driver already
      * keeps a valid page next to the LIMIT and we could potentially
      * consume the trace data that may have been collected there. But we
      * cannot be really sure it is available, and the TRBPTR may not
      * indicate the same. Also, affected cores are also affected by another
      * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus
      * could potentially pad an entire PAGE_SIZE - 64bytes, to get those
      * 64bytes. Thus we ignore the potential triggering of the erratum
      * on WRAP and limit the data to LIMIT.
      */
     if (wrap)
         write = get_trbe_limit_pointer();
     else
         write = get_trbe_write_pointer();
 
     /*
      * TRBE may use a different base address than the base
      * of the ring buffer. Thus use the beginning of the ring
      * buffer to compute the offsets.
      */
     end_off = write - buf->trbe_base;
     start_off = PERF_IDX2OFF(handle->head, buf);
 
     if (WARN_ON_ONCE(end_off < start_off))
         return 0;
 
     size = end_off - start_off;
     /*
      * If the TRBE is affected by the following erratum, we must fill
      * the space we skipped with IGNORE packets. And we are always
      * guaranteed to have at least a PAGE_SIZE space in the buffer.
      */
     if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&
         !WARN_ON(size < overwrite_skip))
         __trbe_pad_buf(buf, start_off, overwrite_skip);
 
     return size;
 }
 
 static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,
                    struct perf_event *event, void **pages,
                    int nr_pages, bool snapshot)
 {
     struct trbe_buf *buf;
     struct page **pglist;
     int i;
 
     /*
      * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with
      * just a single page, there would not be any room left while writing
      * into a partially filled TRBE buffer after the page size alignment.
      * Hence restrict the minimum buffer size as two pages.
      */
     if (nr_pages < 2)
         return NULL;
 
     buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event));
     if (!buf)
         return ERR_PTR(-ENOMEM);
 
     pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
     if (!pglist) {
         kfree(buf);
         return ERR_PTR(-ENOMEM);
     }
 
     for (i = 0; i < nr_pages; i++)
         pglist[i] = virt_to_page(pages[i]);
 
     buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
     if (!buf->trbe_base) {
         kfree(pglist);
         kfree(buf);
         return ERR_PTR(-ENOMEM);
     }
     buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;
     buf->trbe_write = buf->trbe_base;
     buf->snapshot = snapshot;
     buf->nr_pages = nr_pages;
     buf->pages = pages;
     kfree(pglist);
     return buf;
 }
 
 static void arm_trbe_free_buffer(void *config)
 {
     struct trbe_buf *buf = config;
 
     vunmap((void *)buf->trbe_base);
     kfree(buf);
 }
 
 static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,
                         struct perf_output_handle *handle,
                         void *config)
 {
     struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
     struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
     struct trbe_buf *buf = config;
     enum trbe_fault_action act;
     unsigned long size, status;
     unsigned long flags;
     bool wrap = false;
 
     WARN_ON(buf->cpudata != cpudata);
     WARN_ON(cpudata->cpu != smp_processor_id());
     WARN_ON(cpudata->drvdata != drvdata);
     if (cpudata->mode != CS_MODE_PERF)
         return 0;
 
     /*
      * We are about to disable the TRBE. And this could in turn
      * fill up the buffer triggering, an IRQ. This could be consumed
      * by the PE asynchronously, causing a race here against
      * the IRQ handler in closing out the handle. So, let us
      * make sure the IRQ can't trigger while we are collecting
      * the buffer. We also make sure that a WRAP event is handled
      * accordingly.
      */
     local_irq_save(flags);
 
     /*
      * If the TRBE was disabled due to lack of space in the AUX buffer or a
      * spurious fault, the driver leaves it disabled, truncating the buffer.
      * Since the etm_perf driver expects to close out the AUX buffer, the
      * driver skips it. Thus, just pass in 0 size here to indicate that the
      * buffer was truncated.
      */
     if (!is_trbe_enabled()) {
         size = 0;
         goto done;
     }
     /*
      * perf handle structure needs to be shared with the TRBE IRQ handler for
      * capturing trace data and restarting the handle. There is a probability
      * of an undefined reference based crash when etm event is being stopped
      * while a TRBE IRQ also getting processed. This happens due the release
      * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping
      * the TRBE here will ensure that no IRQ could be generated when the perf
      * handle gets freed in etm_event_stop().
      */
     trbe_drain_and_disable_local(cpudata);
 
     /* Check if there is a pending interrupt and handle it here */
     status = read_sysreg_s(SYS_TRBSR_EL1);
     if (is_trbe_irq(status)) {
 
         /*
          * Now that we are handling the IRQ here, clear the IRQ
          * from the status, to let the irq handler know that it
          * is taken care of.
          */
         clr_trbe_irq();
         isb();
 
         act = trbe_get_fault_act(handle, status);
         /*
          * If this was not due to a WRAP event, we have some
          * errors and as such buffer is empty.
          */
         if (act != TRBE_FAULT_ACT_WRAP) {
             size = 0;
             goto done;
         }
 
         trbe_report_wrap_event(handle);
         wrap = true;
     }
 
     size = trbe_get_trace_size(handle, buf, wrap);
 
 done:
     local_irq_restore(flags);
 
     if (buf->snapshot)
         handle->head += size;
     return size;
 }
 
 
 static int trbe_apply_work_around_before_enable(struct trbe_buf *buf)
 {
     /*
      * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache
      * line size from the "TRBBASER_EL1" in the event of a "FILL".
      * Thus, we could loose some amount of the trace at the base.
      *
      * Before Fix:
      *
      *  normal-BASE     head (normal-TRBPTR)         tail (normal-LIMIT)
      *  |                   \/                       /
      *   -------------------------------------------------------------
      *  |   Pg0      |   Pg1       |           |          |  PgN     |
      *   -------------------------------------------------------------
      *
      * In the normal course of action, we would set the TRBBASER to the
      * beginning of the ring-buffer (normal-BASE). But with the erratum,
      * the TRBE could overwrite the contents at the "normal-BASE", after
      * hitting the "normal-LIMIT", since it doesn't stop as expected. And
      * this is wrong. This could result in overwriting trace collected in
      * one of the previous runs, being consumed by the user. So we must
      * always make sure that the TRBBASER is within the region
      * [head, head+size]. Note that TRBBASER must be PAGE aligned,
      *
      *  After moving the BASE:
      *
      *  normal-BASE     head (normal-TRBPTR)         tail (normal-LIMIT)
      *  |                   \/                       /
      *   -------------------------------------------------------------
      *  |         |          |xyzdef.     |..   tuvw|                |
      *   -------------------------------------------------------------
      *                      /
      *              New-BASER
      *
      * Also, we would set the TRBPTR to head (after adjusting for
      * alignment) at normal-PTR. This would mean that the last few bytes
      * of the trace (say, "xyz") might overwrite the first few bytes of
      * trace written ("abc"). More importantly they will appear in what
      * userspace sees as the beginning of the trace, which is wrong. We may
      * not always have space to move the latest trace "xyz" to the correct
      * order as it must appear beyond the LIMIT. (i.e, [head..head+size]).
      * Thus it is easier to ignore those bytes than to complicate the
      * driver to move it, assuming that the erratum was triggered and
      * doing additional checks to see if there is indeed allowed space at
      * TRBLIMITR.LIMIT.
      *
      *  Thus the full workaround will move the BASE and the PTR and would
      *  look like (after padding at the skipped bytes at the end of
      *  session) :
      *
      *  normal-BASE     head (normal-TRBPTR)         tail (normal-LIMIT)
      *  |                   \/                       /
      *   -------------------------------------------------------------
      *  |         |          |///abc..     |..  rst|                |
      *   -------------------------------------------------------------
      *                      /    |
      *              New-BASER    New-TRBPTR
      *
      * To summarize, with the work around:
      *
      *  - We always align the offset for the next session to PAGE_SIZE
      *    (This is to ensure we can program the TRBBASER to this offset
      *    within the region [head...head+size]).
      *
      *  - At TRBE enable:
      *     - Set the TRBBASER to the page aligned offset of the current
      *       proposed write offset. (which is guaranteed to be aligned
      *       as above)
      *     - Move the TRBPTR to skip first 256bytes (that might be
      *       overwritten with the erratum). This ensures that the trace
      *       generated in the session is not re-written.
      *
      *  - At trace collection:
      *     - Pad the 256bytes skipped above again with IGNORE packets.
      */
     if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {
         if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE)))
             return -EINVAL;
         buf->trbe_hw_base = buf->trbe_write;
         buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
     }
 
     /*
      * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to
      * the next page after the TRBLIMITR.LIMIT. For perf, the "next page"
      * may be:
      *     - The page beyond the ring buffer. This could mean, TRBE could
      *       corrupt another entity (kernel / user)
      *     - A portion of the "ring buffer" consumed by the userspace.
      *       i.e, a page outisde [head, head + size].
      *
      * We work around this by:
      *     - Making sure that we have at least an extra space of PAGE left
      *       in the ring buffer [head, head + size], than we normally do
      *       without the erratum. See trbe_min_trace_buf_size().
      *
      *     - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside
      *       the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ).
      */
     if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {
         s64 space = buf->trbe_limit - buf->trbe_write;
         /*
          * We must have more than a PAGE_SIZE worth space in the proposed
          * range for the TRBE.
          */
         if (WARN_ON(space <= PAGE_SIZE ||
                 !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE)))
             return -EINVAL;
         buf->trbe_limit -= PAGE_SIZE;
     }
 
     return 0;
 }
 
 static int __arm_trbe_enable(struct trbe_buf *buf,
                  struct perf_output_handle *handle)
 {
     int ret = 0;
 
     perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
     buf->trbe_limit = compute_trbe_buffer_limit(handle);
     buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);
     if (buf->trbe_limit == buf->trbe_base) {
         ret = -ENOSPC;
         goto err;
     }
     /* Set the base of the TRBE to the buffer base */
     buf->trbe_hw_base = buf->trbe_base;
 
     ret = trbe_apply_work_around_before_enable(buf);
     if (ret)
         goto err;
 
     *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;
     trbe_enable_hw(buf);
     return 0;
 err:
     trbe_stop_and_truncate_event(handle);
     return ret;
 }
 
 static int arm_trbe_enable(struct coresight_device *csdev, u32 mode, void *data)
 {
     struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
     struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
     struct perf_output_handle *handle = data;
     struct trbe_buf *buf = etm_perf_sink_config(handle);
 
     WARN_ON(cpudata->cpu != smp_processor_id());
     WARN_ON(cpudata->drvdata != drvdata);
     if (mode != CS_MODE_PERF)
         return -EINVAL;
 
     cpudata->buf = buf;
     cpudata->mode = mode;
     buf->cpudata = cpudata;
 
     return __arm_trbe_enable(buf, handle);
 }
 
 static int arm_trbe_disable(struct coresight_device *csdev)
 {
     struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
     struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
     struct trbe_buf *buf = cpudata->buf;
 
     WARN_ON(buf->cpudata != cpudata);
     WARN_ON(cpudata->cpu != smp_processor_id());
     WARN_ON(cpudata->drvdata != drvdata);
     if (cpudata->mode != CS_MODE_PERF)
         return -EINVAL;
 
     trbe_drain_and_disable_local(cpudata);
     buf->cpudata = NULL;
     cpudata->buf = NULL;
     cpudata->mode = CS_MODE_DISABLED;
     return 0;
 }
 
 static void trbe_handle_spurious(struct perf_output_handle *handle)
 {
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
 
     /*
      * If the IRQ was spurious, simply re-enable the TRBE
      * back without modifying the buffer parameters to
      * retain the trace collected so far.
      */
     set_trbe_enabled(buf->cpudata, trblimitr);
 }
 
 static int trbe_handle_overflow(struct perf_output_handle *handle)
 {
     struct perf_event *event = handle->event;
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     unsigned long size;
     struct etm_event_data *event_data;
 
     size = trbe_get_trace_size(handle, buf, true);
     if (buf->snapshot)
         handle->head += size;
 
     trbe_report_wrap_event(handle);
     perf_aux_output_end(handle, size);
     event_data = perf_aux_output_begin(handle, event);
     if (!event_data) {
         /*
          * We are unable to restart the trace collection,
          * thus leave the TRBE disabled. The etm-perf driver
          * is able to detect this with a disconnected handle
          * (handle->event = NULL).
          */
         trbe_drain_and_disable_local(buf->cpudata);
         *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
         return -EINVAL;
     }
 
     return __arm_trbe_enable(buf, handle);
 }
 
 static bool is_perf_trbe(struct perf_output_handle *handle)
 {
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     struct trbe_cpudata *cpudata = buf->cpudata;
     struct trbe_drvdata *drvdata = cpudata->drvdata;
     int cpu = smp_processor_id();
 
     WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());
     WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());
 
     if (cpudata->mode != CS_MODE_PERF)
         return false;
 
     if (cpudata->cpu != cpu)
         return false;
 
     if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus))
         return false;
 
     return true;
 }
 
 static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
 {
     struct perf_output_handle **handle_ptr = dev;
     struct perf_output_handle *handle = *handle_ptr;
     struct trbe_buf *buf = etm_perf_sink_config(handle);
     enum trbe_fault_action act;
     u64 status;
     bool truncated = false;
     u64 trfcr;
 
     /* Reads to TRBSR_EL1 is fine when TRBE is active */
     status = read_sysreg_s(SYS_TRBSR_EL1);
     /*
      * If the pending IRQ was handled by update_buffer callback
      * we have nothing to do here.
      */
     if (!is_trbe_irq(status))
         return IRQ_NONE;
 
     /* Prohibit the CPU from tracing before we disable the TRBE */
     trfcr = cpu_prohibit_trace();
     /*
      * Ensure the trace is visible to the CPUs and
      * any external aborts have been resolved.
      */
     trbe_drain_and_disable_local(buf->cpudata);
     clr_trbe_irq();
     isb();
 
     if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))
         return IRQ_NONE;
 
     if (!is_perf_trbe(handle))
         return IRQ_NONE;
 
     act = trbe_get_fault_act(handle, status);
     switch (act) {
     case TRBE_FAULT_ACT_WRAP:
         truncated = !!trbe_handle_overflow(handle);
         break;
     case TRBE_FAULT_ACT_SPURIOUS:
         trbe_handle_spurious(handle);
         break;
     case TRBE_FAULT_ACT_FATAL:
         trbe_stop_and_truncate_event(handle);
         truncated = true;
         break;
     }
 
     /*
      * If the buffer was truncated, ensure perf callbacks
      * have completed, which will disable the event.
      *
      * Otherwise, restore the trace filter controls to
      * allow the tracing.
      */
     if (truncated)
         irq_work_run();
     else
         write_trfcr(trfcr);
 
     return IRQ_HANDLED;
 }
 
 static const struct coresight_ops_sink arm_trbe_sink_ops = {
     .enable     = arm_trbe_enable,
     .disable    = arm_trbe_disable,
     .alloc_buffer   = arm_trbe_alloc_buffer,
     .free_buffer    = arm_trbe_free_buffer,
     .update_buffer  = arm_trbe_update_buffer,
 };
 
 static const struct coresight_ops arm_trbe_cs_ops = {
     .sink_ops   = &arm_trbe_sink_ops,
 };
 
 static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
     struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
 
     return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);
 }
 static DEVICE_ATTR_RO(align);
 
 static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
     struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
 
     return sprintf(buf, "%d\n", cpudata->trbe_flag);
 }
 static DEVICE_ATTR_RO(flag);
 
 static struct attribute *arm_trbe_attrs[] = {
     &dev_attr_align.attr,
     &dev_attr_flag.attr,
     NULL,
 };
 
 static const struct attribute_group arm_trbe_group = {
     .attrs = arm_trbe_attrs,
 };
 
 static const struct attribute_group *arm_trbe_groups[] = {
     &arm_trbe_group,
     NULL,
 };
 
 static void arm_trbe_enable_cpu(void *info)
 {
     struct trbe_drvdata *drvdata = info;
     struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
 
     trbe_reset_local(cpudata);
     enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE);
 }
 
 static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
 {
     struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
     struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
     struct coresight_desc desc = { 0 };
     struct device *dev;
 
     if (WARN_ON(trbe_csdev))
         return;
 
     /* If the TRBE was not probed on the CPU, we shouldn't be here */
     if (WARN_ON(!cpudata->drvdata))
         return;
 
     dev = &cpudata->drvdata->pdev->dev;
     desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
     if (!desc.name)
         goto cpu_clear;
 
     desc.type = CORESIGHT_DEV_TYPE_SINK;
     desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;
     desc.ops = &arm_trbe_cs_ops;
     desc.pdata = dev_get_platdata(dev);
     desc.groups = arm_trbe_groups;
     desc.dev = dev;
     trbe_csdev = coresight_register(&desc);
     if (IS_ERR(trbe_csdev))
         goto cpu_clear;
 
     dev_set_drvdata(&trbe_csdev->dev, cpudata);
     coresight_set_percpu_sink(cpu, trbe_csdev);
     return;
 cpu_clear:
     cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
 }
 
 /*
  * Must be called with preemption disabled, for trbe_check_errata().
  */
 static void arm_trbe_probe_cpu(void *info)
 {
     struct trbe_drvdata *drvdata = info;
     int cpu = smp_processor_id();
     struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
     u64 trbidr;
 
     if (WARN_ON(!cpudata))
         goto cpu_clear;
 
     if (!is_trbe_available()) {
         pr_err("TRBE is not implemented on cpu %d\n", cpu);
         goto cpu_clear;
     }
 
     trbidr = read_sysreg_s(SYS_TRBIDR_EL1);
     if (!is_trbe_programmable(trbidr)) {
         pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);
         goto cpu_clear;
     }
 
     cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);
     if (cpudata->trbe_hw_align > SZ_2K) {
         pr_err("Unsupported alignment on cpu %d\n", cpu);
         goto cpu_clear;
     }
 
     /*
      * Run the TRBE erratum checks, now that we know
      * this instance is about to be registered.
      */
     trbe_check_errata(cpudata);
 
     if (trbe_is_broken(cpudata)) {
         pr_err("Disabling TRBE on cpu%d due to erratum\n", cpu);
         goto cpu_clear;
     }
 
     /*
      * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
      * we must always program the TBRPTR_EL1, 256bytes from a page
      * boundary, with TRBBASER_EL1 set to the page, to prevent
      * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event.
      *
      * Thus make sure we always align our write pointer to a PAGE_SIZE,
      * which also guarantees that we have at least a PAGE_SIZE space in
      * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip
      * the required bytes at the base.
      */
     if (trbe_may_overwrite_in_fill_mode(cpudata))
         cpudata->trbe_align = PAGE_SIZE;
     else
         cpudata->trbe_align = cpudata->trbe_hw_align;
 
     cpudata->trbe_flag = get_trbe_flag_update(trbidr);
     cpudata->cpu = cpu;
     cpudata->drvdata = drvdata;
     return;
 cpu_clear:
     cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
 }
 
 static void arm_trbe_remove_coresight_cpu(void *info)
 {
     int cpu = smp_processor_id();
     struct trbe_drvdata *drvdata = info;
     struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
     struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
 
     disable_percpu_irq(drvdata->irq);
     trbe_reset_local(cpudata);
     if (trbe_csdev) {
         coresight_unregister(trbe_csdev);
         cpudata->drvdata = NULL;
         coresight_set_percpu_sink(cpu, NULL);
     }
 }
 
 static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
 {
     int cpu;
 
     drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));
     if (!drvdata->cpudata)
         return -ENOMEM;
 
     for_each_cpu(cpu, &drvdata->supported_cpus) {
         /* If we fail to probe the CPU, let us defer it to hotplug callbacks */
         if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))
             continue;
         if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
             arm_trbe_register_coresight_cpu(drvdata, cpu);
         if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
             smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1);
     }
     return 0;
 }
 
 static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)
 {
     int cpu;
 
     for_each_cpu(cpu, &drvdata->supported_cpus)
         smp_call_function_single(cpu, arm_trbe_remove_coresight_cpu, drvdata, 1);
     free_percpu(drvdata->cpudata);
     return 0;
 }
 
 static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)
 {
     preempt_disable();
     arm_trbe_probe_cpu(drvdata);
     preempt_enable();
 }
 
 static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)
 {
     struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
 
     if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
 
         /*
          * If this CPU was not probed for TRBE,
          * initialize it now.
          */
         if (!coresight_get_percpu_sink(cpu)) {
             arm_trbe_probe_hotplugged_cpu(drvdata);
             if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
                 arm_trbe_register_coresight_cpu(drvdata, cpu);
             if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
                 arm_trbe_enable_cpu(drvdata);
         } else {
             arm_trbe_enable_cpu(drvdata);
         }
     }
     return 0;
 }
 
 static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)
 {
     struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
 
     if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
         struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
 
         disable_percpu_irq(drvdata->irq);
         trbe_reset_local(cpudata);
     }
     return 0;
 }
 
 static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)
 {
     enum cpuhp_state trbe_online;
     int ret;
 
     trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
                           arm_trbe_cpu_startup, arm_trbe_cpu_teardown);
     if (trbe_online < 0)
         return trbe_online;
 
     ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node);
     if (ret) {
         cpuhp_remove_multi_state(trbe_online);
         return ret;
     }
     drvdata->trbe_online = trbe_online;
     return 0;
 }
 
 static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)
 {
     cpuhp_remove_multi_state(drvdata->trbe_online);
 }
 
 static int arm_trbe_probe_irq(struct platform_device *pdev,
                   struct trbe_drvdata *drvdata)
 {
     int ret;
 
     drvdata->irq = platform_get_irq(pdev, 0);
     if (drvdata->irq < 0) {
         pr_err("IRQ not found for the platform device\n");
         return drvdata->irq;
     }
 
     if (!irq_is_percpu(drvdata->irq)) {
         pr_err("IRQ is not a PPI\n");
         return -EINVAL;
     }
 
     if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus))
         return -EINVAL;
 
     drvdata->handle = alloc_percpu(struct perf_output_handle *);
     if (!drvdata->handle)
         return -ENOMEM;
 
     ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);
     if (ret) {
         free_percpu(drvdata->handle);
         return ret;
     }
     return 0;
 }
 
 static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)
 {
     free_percpu_irq(drvdata->irq, drvdata->handle);
     free_percpu(drvdata->handle);
 }
 
 static int arm_trbe_device_probe(struct platform_device *pdev)
 {
     struct coresight_platform_data *pdata;
     struct trbe_drvdata *drvdata;
     struct device *dev = &pdev->dev;
     int ret;
 
     /* Trace capture is not possible with kernel page table isolation */
     if (arm64_kernel_unmapped_at_el0()) {
         pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");
         return -EOPNOTSUPP;
     }
 
     drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
     if (!drvdata)
         return -ENOMEM;
 
     pdata = coresight_get_platform_data(dev);
     if (IS_ERR(pdata))
         return PTR_ERR(pdata);
 
     dev_set_drvdata(dev, drvdata);
     dev->platform_data = pdata;
     drvdata->pdev = pdev;
     ret = arm_trbe_probe_irq(pdev, drvdata);
     if (ret)
         return ret;
 
     ret = arm_trbe_probe_coresight(drvdata);
     if (ret)
         goto probe_failed;
 
     ret = arm_trbe_probe_cpuhp(drvdata);
     if (ret)
         goto cpuhp_failed;
 
     return 0;
 cpuhp_failed:
     arm_trbe_remove_coresight(drvdata);
 probe_failed:
     arm_trbe_remove_irq(drvdata);
     return ret;
 }
 
 static int arm_trbe_device_remove(struct platform_device *pdev)
 {
     struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
 
     arm_trbe_remove_cpuhp(drvdata);
     arm_trbe_remove_coresight(drvdata);
     arm_trbe_remove_irq(drvdata);
     return 0;
 }
 
 static const struct of_device_id arm_trbe_of_match[] = {
     { .compatible = "arm,trace-buffer-extension"},
     {},
 };
 MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
 
 static struct platform_driver arm_trbe_driver = {
     .driver = {
         .name = DRVNAME,
         .of_match_table = of_match_ptr(arm_trbe_of_match),
         .suppress_bind_attrs = true,
     },
     .probe  = arm_trbe_device_probe,
     .remove = arm_trbe_device_remove,
 };
 
 static int __init arm_trbe_init(void)
 {
     int ret;
 
     ret = platform_driver_register(&arm_trbe_driver);
     if (!ret)
         return 0;
 
     pr_err("Error registering %s platform driver\n", DRVNAME);
     return ret;
 }
 
 static void __exit arm_trbe_exit(void)
 {
     platform_driver_unregister(&arm_trbe_driver);
 }
 module_init(arm_trbe_init);
 module_exit(arm_trbe_exit);
 
 MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");
 MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");
 MODULE_LICENSE("GPL v2");

This page was automatically generated by the 2.1.0 LXR engine. The LXR team