OSCL-LXR linux-6.0.1/​kernel/​irq/​timings.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0
 // Copyright (C) 2016, Linaro Ltd - Daniel Lezcano <daniel.lezcano@linaro.org>
 #define pr_fmt(fmt) "irq_timings: " fmt
 
 #include <linux/kernel.h>
 #include <linux/percpu.h>
 #include <linux/slab.h>
 #include <linux/static_key.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/idr.h>
 #include <linux/irq.h>
 #include <linux/math64.h>
 #include <linux/log2.h>
 
 #include <trace/events/irq.h>
 
 #include "internals.h"
 
 DEFINE_STATIC_KEY_FALSE(irq_timing_enabled);
 
 DEFINE_PER_CPU(struct irq_timings, irq_timings);
 
 static DEFINE_IDR(irqt_stats);
 
 void irq_timings_enable(void)
 {
     static_branch_enable(&irq_timing_enabled);
 }
 
 void irq_timings_disable(void)
 {
     static_branch_disable(&irq_timing_enabled);
 }
 
 /*
  * The main goal of this algorithm is to predict the next interrupt
  * occurrence on the current CPU.
  *
  * Currently, the interrupt timings are stored in a circular array
  * buffer every time there is an interrupt, as a tuple: the interrupt
  * number and the associated timestamp when the event occurred <irq,
  * timestamp>.
  *
  * For every interrupt occurring in a short period of time, we can
  * measure the elapsed time between the occurrences for the same
  * interrupt and we end up with a suite of intervals. The experience
  * showed the interrupts are often coming following a periodic
  * pattern.
  *
  * The objective of the algorithm is to find out this periodic pattern
  * in a fastest way and use its period to predict the next irq event.
  *
  * When the next interrupt event is requested, we are in the situation
  * where the interrupts are disabled and the circular buffer
  * containing the timings is filled with the events which happened
  * after the previous next-interrupt-event request.
  *
  * At this point, we read the circular buffer and we fill the irq
  * related statistics structure. After this step, the circular array
  * containing the timings is empty because all the values are
  * dispatched in their corresponding buffers.
  *
  * Now for each interrupt, we can predict the next event by using the
  * suffix array, log interval and exponential moving average
  *
  * 1. Suffix array
  *
  * Suffix array is an array of all the suffixes of a string. It is
  * widely used as a data structure for compression, text search, ...
  * For instance for the word 'banana', the suffixes will be: 'banana'
  * 'anana' 'nana' 'ana' 'na' 'a'
  *
  * Usually, the suffix array is sorted but for our purpose it is
  * not necessary and won't provide any improvement in the context of
  * the solved problem where we clearly define the boundaries of the
  * search by a max period and min period.
  *
  * The suffix array will build a suite of intervals of different
  * length and will look for the repetition of each suite. If the suite
  * is repeating then we have the period because it is the length of
  * the suite whatever its position in the buffer.
  *
  * 2. Log interval
  *
  * We saw the irq timings allow to compute the interval of the
  * occurrences for a specific interrupt. We can reasonably assume the
  * longer is the interval, the higher is the error for the next event
  * and we can consider storing those interval values into an array
  * where each slot in the array correspond to an interval at the power
  * of 2 of the index. For example, index 12 will contain values
  * between 2^11 and 2^12.
  *
  * At the end we have an array of values where at each index defines a
  * [2^index - 1, 2 ^ index] interval values allowing to store a large
  * number of values inside a small array.
  *
  * For example, if we have the value 1123, then we store it at
  * ilog2(1123) = 10 index value.
  *
  * Storing those value at the specific index is done by computing an
  * exponential moving average for this specific slot. For instance,
  * for values 1800, 1123, 1453, ... fall under the same slot (10) and
  * the exponential moving average is computed every time a new value
  * is stored at this slot.
  *
  * 3. Exponential Moving Average
  *
  * The EMA is largely used to track a signal for stocks or as a low
  * pass filter. The magic of the formula, is it is very simple and the
  * reactivity of the average can be tuned with the factors called
  * alpha.
  *
  * The higher the alphas are, the faster the average respond to the
  * signal change. In our case, if a slot in the array is a big
  * interval, we can have numbers with a big difference between
  * them. The impact of those differences in the average computation
  * can be tuned by changing the alpha value.
  *
  *
  *  -- The algorithm --
  *
  * We saw the different processing above, now let's see how they are
  * used together.
  *
  * For each interrupt:
  *  For each interval:
  *      Compute the index = ilog2(interval)
  *      Compute a new_ema(buffer[index], interval)
  *      Store the index in a circular buffer
  *
  *  Compute the suffix array of the indexes
  *
  *  For each suffix:
  *      If the suffix is reverse-found 3 times
  *          Return suffix
  *
  *  Return Not found
  *
  * However we can not have endless suffix array to be build, it won't
  * make sense and it will add an extra overhead, so we can restrict
  * this to a maximum suffix length of 5 and a minimum suffix length of
  * 2. The experience showed 5 is the majority of the maximum pattern
  * period found for different devices.
  *
  * The result is a pattern finding less than 1us for an interrupt.
  *
  * Example based on real values:
  *
  * Example 1 : MMC write/read interrupt interval:
  *
  *  223947, 1240, 1384, 1386, 1386,
  *  217416, 1236, 1384, 1386, 1387,
  *  214719, 1241, 1386, 1387, 1384,
  *  213696, 1234, 1384, 1386, 1388,
  *  219904, 1240, 1385, 1389, 1385,
  *  212240, 1240, 1386, 1386, 1386,
  *  214415, 1236, 1384, 1386, 1387,
  *  214276, 1234, 1384, 1388, ?
  *
  * For each element, apply ilog2(value)
  *
  *  15, 8, 8, 8, 8,
  *  15, 8, 8, 8, 8,
  *  15, 8, 8, 8, 8,
  *  15, 8, 8, 8, 8,
  *  15, 8, 8, 8, 8,
  *  15, 8, 8, 8, 8,
  *  15, 8, 8, 8, 8,
  *  15, 8, 8, 8, ?
  *
  * Max period of 5, we take the last (max_period * 3) 15 elements as
  * we can be confident if the pattern repeats itself three times it is
  * a repeating pattern.
  *
  *               8,
  *  15, 8, 8, 8, 8,
  *  15, 8, 8, 8, 8,
  *  15, 8, 8, 8, ?
  *
  * Suffixes are:
  *
  *  1) 8, 15, 8, 8, 8  <- max period
  *  2) 8, 15, 8, 8
  *  3) 8, 15, 8
  *  4) 8, 15           <- min period
  *
  * From there we search the repeating pattern for each suffix.
  *
  * buffer: 8, 15, 8, 8, 8, 8, 15, 8, 8, 8, 8, 15, 8, 8, 8
  *         |   |  |  |  |  |   |  |  |  |  |   |  |  |  |
  *         8, 15, 8, 8, 8  |   |  |  |  |  |   |  |  |  |
  *                         8, 15, 8, 8, 8  |   |  |  |  |
  *                                         8, 15, 8, 8, 8
  *
  * When moving the suffix, we found exactly 3 matches.
  *
  * The first suffix with period 5 is repeating.
  *
  * The next event is (3 * max_period) % suffix_period
  *
  * In this example, the result 0, so the next event is suffix[0] => 8
  *
  * However, 8 is the index in the array of exponential moving average
  * which was calculated on the fly when storing the values, so the
  * interval is ema[8] = 1366
  *
  *
  * Example 2:
  *
  *  4, 3, 5, 100,
  *  3, 3, 5, 117,
  *  4, 4, 5, 112,
  *  4, 3, 4, 110,
  *  3, 5, 3, 117,
  *  4, 4, 5, 112,
  *  4, 3, 4, 110,
  *  3, 4, 5, 112,
  *  4, 3, 4, 110
  *
  * ilog2
  *
  *  0, 0, 0, 4,
  *  0, 0, 0, 4,
  *  0, 0, 0, 4,
  *  0, 0, 0, 4,
  *  0, 0, 0, 4,
  *  0, 0, 0, 4,
  *  0, 0, 0, 4,
  *  0, 0, 0, 4,
  *  0, 0, 0, 4
  *
  * Max period 5:
  *     0, 0, 4,
  *  0, 0, 0, 4,
  *  0, 0, 0, 4,
  *  0, 0, 0, 4
  *
  * Suffixes:
  *
  *  1) 0, 0, 4, 0, 0
  *  2) 0, 0, 4, 0
  *  3) 0, 0, 4
  *  4) 0, 0
  *
  * buffer: 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4
  *         |  |  |  |  |  |  X
  *         0, 0, 4, 0, 0, |  X
  *                        0, 0
  *
  * buffer: 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4
  *         |  |  |  |  |  |  |  |  |  |  |  |  |  |  |
  *         0, 0, 4, 0, |  |  |  |  |  |  |  |  |  |  |
  *                     0, 0, 4, 0, |  |  |  |  |  |  |
  *                                 0, 0, 4, 0, |  |  |
  *                                             0  0  4
  *
  * Pattern is found 3 times, the remaining is 1 which results from
  * (max_period * 3) % suffix_period. This value is the index in the
  * suffix arrays. The suffix array for a period 4 has the value 4
  * at index 1.
  */
 #define EMA_ALPHA_VAL       64
 #define EMA_ALPHA_SHIFT     7
 
 #define PREDICTION_PERIOD_MIN   3
 #define PREDICTION_PERIOD_MAX   5
 #define PREDICTION_FACTOR   4
 #define PREDICTION_MAX      10 /* 2 ^ PREDICTION_MAX useconds */
 #define PREDICTION_BUFFER_SIZE  16 /* slots for EMAs, hardly more than 16 */
 
 /*
  * Number of elements in the circular buffer: If it happens it was
  * flushed before, then the number of elements could be smaller than
  * IRQ_TIMINGS_SIZE, so the count is used, otherwise the array size is
  * used as we wrapped. The index begins from zero when we did not
  * wrap. That could be done in a nicer way with the proper circular
  * array structure type but with the cost of extra computation in the
  * interrupt handler hot path. We choose efficiency.
  */
 #define for_each_irqts(i, irqts)                    \
     for (i = irqts->count < IRQ_TIMINGS_SIZE ?          \
              0 : irqts->count & IRQ_TIMINGS_MASK,       \
              irqts->count = min(IRQ_TIMINGS_SIZE,       \
                     irqts->count);          \
          irqts->count > 0; irqts->count--,              \
              i = (i + 1) & IRQ_TIMINGS_MASK)
 
 struct irqt_stat {
     u64 last_ts;
     u64 ema_time[PREDICTION_BUFFER_SIZE];
     int timings[IRQ_TIMINGS_SIZE];
     int circ_timings[IRQ_TIMINGS_SIZE];
     int count;
 };
 
 /*
  * Exponential moving average computation
  */
 static u64 irq_timings_ema_new(u64 value, u64 ema_old)
 {
     s64 diff;
 
     if (unlikely(!ema_old))
         return value;
 
     diff = (value - ema_old) * EMA_ALPHA_VAL;
     /*
      * We can use a s64 type variable to be added with the u64
      * ema_old variable as this one will never have its topmost
      * bit set, it will be always smaller than 2^63 nanosec
      * interrupt interval (292 years).
      */
     return ema_old + (diff >> EMA_ALPHA_SHIFT);
 }
 
 static int irq_timings_next_event_index(int *buffer, size_t len, int period_max)
 {
     int period;
 
     /*
      * Move the beginning pointer to the end minus the max period x 3.
      * We are at the point we can begin searching the pattern
      */
     buffer = &buffer[len - (period_max * 3)];
 
     /* Adjust the length to the maximum allowed period x 3 */
     len = period_max * 3;
 
     /*
      * The buffer contains the suite of intervals, in a ilog2
      * basis, we are looking for a repetition. We point the
      * beginning of the search three times the length of the
      * period beginning at the end of the buffer. We do that for
      * each suffix.
      */
     for (period = period_max; period >= PREDICTION_PERIOD_MIN; period--) {
 
         /*
          * The first comparison always succeed because the
          * suffix is deduced from the first n-period bytes of
          * the buffer and we compare the initial suffix with
          * itself, so we can skip the first iteration.
          */
         int idx = period;
         size_t size = period;
 
         /*
          * We look if the suite with period 'i' repeat
          * itself. If it is truncated at the end, as it
          * repeats we can use the period to find out the next
          * element with the modulo.
          */
         while (!memcmp(buffer, &buffer[idx], size * sizeof(int))) {
 
             /*
              * Move the index in a period basis
              */
             idx += size;
 
             /*
              * If this condition is reached, all previous
              * memcmp were successful, so the period is
              * found.
              */
             if (idx == len)
                 return buffer[len % period];
 
             /*
              * If the remaining elements to compare are
              * smaller than the period, readjust the size
              * of the comparison for the last iteration.
              */
             if (len - idx < period)
                 size = len - idx;
         }
     }
 
     return -1;
 }
 
 static u64 __irq_timings_next_event(struct irqt_stat *irqs, int irq, u64 now)
 {
     int index, i, period_max, count, start, min = INT_MAX;
 
     if ((now - irqs->last_ts) >= NSEC_PER_SEC) {
         irqs->count = irqs->last_ts = 0;
         return U64_MAX;
     }
 
     /*
      * As we want to find three times the repetition, we need a
      * number of intervals greater or equal to three times the
      * maximum period, otherwise we truncate the max period.
      */
     period_max = irqs->count > (3 * PREDICTION_PERIOD_MAX) ?
         PREDICTION_PERIOD_MAX : irqs->count / 3;
 
     /*
      * If we don't have enough irq timings for this prediction,
      * just bail out.
      */
     if (period_max <= PREDICTION_PERIOD_MIN)
         return U64_MAX;
 
     /*
      * 'count' will depends if the circular buffer wrapped or not
      */
     count = irqs->count < IRQ_TIMINGS_SIZE ?
         irqs->count : IRQ_TIMINGS_SIZE;
 
     start = irqs->count < IRQ_TIMINGS_SIZE ?
         0 : (irqs->count & IRQ_TIMINGS_MASK);
 
     /*
      * Copy the content of the circular buffer into another buffer
      * in order to linearize the buffer instead of dealing with
      * wrapping indexes and shifted array which will be prone to
      * error and extremely difficult to debug.
      */
     for (i = 0; i < count; i++) {
         int index = (start + i) & IRQ_TIMINGS_MASK;
 
         irqs->timings[i] = irqs->circ_timings[index];
         min = min_t(int, irqs->timings[i], min);
     }
 
     index = irq_timings_next_event_index(irqs->timings, count, period_max);
     if (index < 0)
         return irqs->last_ts + irqs->ema_time[min];
 
     return irqs->last_ts + irqs->ema_time[index];
 }
 
 static __always_inline int irq_timings_interval_index(u64 interval)
 {
     /*
      * The PREDICTION_FACTOR increase the interval size for the
      * array of exponential average.
      */
     u64 interval_us = (interval >> 10) / PREDICTION_FACTOR;
 
     return likely(interval_us) ? ilog2(interval_us) : 0;
 }
 
 static __always_inline void __irq_timings_store(int irq, struct irqt_stat *irqs,
                         u64 interval)
 {
     int index;
 
     /*
      * Get the index in the ema table for this interrupt.
      */
     index = irq_timings_interval_index(interval);
 
     if (index > PREDICTION_BUFFER_SIZE - 1) {
         irqs->count = 0;
         return;
     }
 
     /*
      * Store the index as an element of the pattern in another
      * circular array.
      */
     irqs->circ_timings[irqs->count & IRQ_TIMINGS_MASK] = index;
 
     irqs->ema_time[index] = irq_timings_ema_new(interval,
                             irqs->ema_time[index]);
 
     irqs->count++;
 }
 
 static inline void irq_timings_store(int irq, struct irqt_stat *irqs, u64 ts)
 {
     u64 old_ts = irqs->last_ts;
     u64 interval;
 
     /*
      * The timestamps are absolute time values, we need to compute
      * the timing interval between two interrupts.
      */
     irqs->last_ts = ts;
 
     /*
      * The interval type is u64 in order to deal with the same
      * type in our computation, that prevent mindfuck issues with
      * overflow, sign and division.
      */
     interval = ts - old_ts;
 
     /*
      * The interrupt triggered more than one second apart, that
      * ends the sequence as predictable for our purpose. In this
      * case, assume we have the beginning of a sequence and the
      * timestamp is the first value. As it is impossible to
      * predict anything at this point, return.
      *
      * Note the first timestamp of the sequence will always fall
      * in this test because the old_ts is zero. That is what we
      * want as we need another timestamp to compute an interval.
      */
     if (interval >= NSEC_PER_SEC) {
         irqs->count = 0;
         return;
     }
 
     __irq_timings_store(irq, irqs, interval);
 }
 
 /**
  * irq_timings_next_event - Return when the next event is supposed to arrive
  *
  * During the last busy cycle, the number of interrupts is incremented
  * and stored in the irq_timings structure. This information is
  * necessary to:
  *
  * - know if the index in the table wrapped up:
  *
  *      If more than the array size interrupts happened during the
  *      last busy/idle cycle, the index wrapped up and we have to
  *      begin with the next element in the array which is the last one
  *      in the sequence, otherwise it is at the index 0.
  *
  * - have an indication of the interrupts activity on this CPU
  *   (eg. irq/sec)
  *
  * The values are 'consumed' after inserting in the statistical model,
  * thus the count is reinitialized.
  *
  * The array of values **must** be browsed in the time direction, the
  * timestamp must increase between an element and the next one.
  *
  * Returns a nanosec time based estimation of the earliest interrupt,
  * U64_MAX otherwise.
  */
 u64 irq_timings_next_event(u64 now)
 {
     struct irq_timings *irqts = this_cpu_ptr(&irq_timings);
     struct irqt_stat *irqs;
     struct irqt_stat __percpu *s;
     u64 ts, next_evt = U64_MAX;
     int i, irq = 0;
 
     /*
      * This function must be called with the local irq disabled in
      * order to prevent the timings circular buffer to be updated
      * while we are reading it.
      */
     lockdep_assert_irqs_disabled();
 
     if (!irqts->count)
         return next_evt;
 
     /*
      * Number of elements in the circular buffer: If it happens it
      * was flushed before, then the number of elements could be
      * smaller than IRQ_TIMINGS_SIZE, so the count is used,
      * otherwise the array size is used as we wrapped. The index
      * begins from zero when we did not wrap. That could be done
      * in a nicer way with the proper circular array structure
      * type but with the cost of extra computation in the
      * interrupt handler hot path. We choose efficiency.
      *
      * Inject measured irq/timestamp to the pattern prediction
      * model while decrementing the counter because we consume the
      * data from our circular buffer.
      */
     for_each_irqts(i, irqts) {
         irq = irq_timing_decode(irqts->values[i], &ts);
         s = idr_find(&irqt_stats, irq);
         if (s)
             irq_timings_store(irq, this_cpu_ptr(s), ts);
     }
 
     /*
      * Look in the list of interrupts' statistics, the earliest
      * next event.
      */
     idr_for_each_entry(&irqt_stats, s, i) {
 
         irqs = this_cpu_ptr(s);
 
         ts = __irq_timings_next_event(irqs, i, now);
         if (ts <= now)
             return now;
 
         if (ts < next_evt)
             next_evt = ts;
     }
 
     return next_evt;
 }
 
 void irq_timings_free(int irq)
 {
     struct irqt_stat __percpu *s;
 
     s = idr_find(&irqt_stats, irq);
     if (s) {
         free_percpu(s);
         idr_remove(&irqt_stats, irq);
     }
 }
 
 int irq_timings_alloc(int irq)
 {
     struct irqt_stat __percpu *s;
     int id;
 
     /*
      * Some platforms can have the same private interrupt per cpu,
      * so this function may be called several times with the
      * same interrupt number. Just bail out in case the per cpu
      * stat structure is already allocated.
      */
     s = idr_find(&irqt_stats, irq);
     if (s)
         return 0;
 
     s = alloc_percpu(*s);
     if (!s)
         return -ENOMEM;
 
     idr_preload(GFP_KERNEL);
     id = idr_alloc(&irqt_stats, s, irq, irq + 1, GFP_NOWAIT);
     idr_preload_end();
 
     if (id < 0) {
         free_percpu(s);
         return id;
     }
 
     return 0;
 }
 
 #ifdef CONFIG_TEST_IRQ_TIMINGS
 struct timings_intervals {
     u64 *intervals;
     size_t count;
 };
 
 /*
  * Intervals are given in nanosecond base
  */
 static u64 intervals0[] __initdata = {
     10000, 50000, 200000, 500000,
     10000, 50000, 200000, 500000,
     10000, 50000, 200000, 500000,
     10000, 50000, 200000, 500000,
     10000, 50000, 200000, 500000,
     10000, 50000, 200000, 500000,
     10000, 50000, 200000, 500000,
     10000, 50000, 200000, 500000,
     10000, 50000, 200000,
 };
 
 static u64 intervals1[] __initdata = {
     223947000, 1240000, 1384000, 1386000, 1386000,
     217416000, 1236000, 1384000, 1386000, 1387000,
     214719000, 1241000, 1386000, 1387000, 1384000,
     213696000, 1234000, 1384000, 1386000, 1388000,
     219904000, 1240000, 1385000, 1389000, 1385000,
     212240000, 1240000, 1386000, 1386000, 1386000,
     214415000, 1236000, 1384000, 1386000, 1387000,
     214276000, 1234000,
 };
 
 static u64 intervals2[] __initdata = {
     4000, 3000, 5000, 100000,
     3000, 3000, 5000, 117000,
     4000, 4000, 5000, 112000,
     4000, 3000, 4000, 110000,
     3000, 5000, 3000, 117000,
     4000, 4000, 5000, 112000,
     4000, 3000, 4000, 110000,
     3000, 4000, 5000, 112000,
     4000,
 };
 
 static u64 intervals3[] __initdata = {
     1385000, 212240000, 1240000,
     1386000, 214415000, 1236000,
     1384000, 214276000, 1234000,
     1386000, 214415000, 1236000,
     1385000, 212240000, 1240000,
     1386000, 214415000, 1236000,
     1384000, 214276000, 1234000,
     1386000, 214415000, 1236000,
     1385000, 212240000, 1240000,
 };
 
 static u64 intervals4[] __initdata = {
     10000, 50000, 10000, 50000,
     10000, 50000, 10000, 50000,
     10000, 50000, 10000, 50000,
     10000, 50000, 10000, 50000,
     10000, 50000, 10000, 50000,
     10000, 50000, 10000, 50000,
     10000, 50000, 10000, 50000,
     10000, 50000, 10000, 50000,
     10000,
 };
 
 static struct timings_intervals tis[] __initdata = {
     { intervals0, ARRAY_SIZE(intervals0) },
     { intervals1, ARRAY_SIZE(intervals1) },
     { intervals2, ARRAY_SIZE(intervals2) },
     { intervals3, ARRAY_SIZE(intervals3) },
     { intervals4, ARRAY_SIZE(intervals4) },
 };
 
 static int __init irq_timings_test_next_index(struct timings_intervals *ti)
 {
     int _buffer[IRQ_TIMINGS_SIZE];
     int buffer[IRQ_TIMINGS_SIZE];
     int index, start, i, count, period_max;
 
     count = ti->count - 1;
 
     period_max = count > (3 * PREDICTION_PERIOD_MAX) ?
         PREDICTION_PERIOD_MAX : count / 3;
 
     /*
      * Inject all values except the last one which will be used
      * to compare with the next index result.
      */
     pr_debug("index suite: ");
 
     for (i = 0; i < count; i++) {
         index = irq_timings_interval_index(ti->intervals[i]);
         _buffer[i & IRQ_TIMINGS_MASK] = index;
         pr_cont("%d ", index);
     }
 
     start = count < IRQ_TIMINGS_SIZE ? 0 :
         count & IRQ_TIMINGS_MASK;
 
     count = min_t(int, count, IRQ_TIMINGS_SIZE);
 
     for (i = 0; i < count; i++) {
         int index = (start + i) & IRQ_TIMINGS_MASK;
         buffer[i] = _buffer[index];
     }
 
     index = irq_timings_next_event_index(buffer, count, period_max);
     i = irq_timings_interval_index(ti->intervals[ti->count - 1]);
 
     if (index != i) {
         pr_err("Expected (%d) and computed (%d) next indexes differ\n",
                i, index);
         return -EINVAL;
     }
 
     return 0;
 }
 
 static int __init irq_timings_next_index_selftest(void)
 {
     int i, ret;
 
     for (i = 0; i < ARRAY_SIZE(tis); i++) {
 
         pr_info("---> Injecting intervals number #%d (count=%zd)\n",
             i, tis[i].count);
 
         ret = irq_timings_test_next_index(&tis[i]);
         if (ret)
             break;
     }
 
     return ret;
 }
 
 static int __init irq_timings_test_irqs(struct timings_intervals *ti)
 {
     struct irqt_stat __percpu *s;
     struct irqt_stat *irqs;
     int i, index, ret, irq = 0xACE5;
 
     ret = irq_timings_alloc(irq);
     if (ret) {
         pr_err("Failed to allocate irq timings\n");
         return ret;
     }
 
     s = idr_find(&irqt_stats, irq);
     if (!s) {
         ret = -EIDRM;
         goto out;
     }
 
     irqs = this_cpu_ptr(s);
 
     for (i = 0; i < ti->count; i++) {
 
         index = irq_timings_interval_index(ti->intervals[i]);
         pr_debug("%d: interval=%llu ema_index=%d\n",
              i, ti->intervals[i], index);
 
         __irq_timings_store(irq, irqs, ti->intervals[i]);
         if (irqs->circ_timings[i & IRQ_TIMINGS_MASK] != index) {
             ret = -EBADSLT;
             pr_err("Failed to store in the circular buffer\n");
             goto out;
         }
     }
 
     if (irqs->count != ti->count) {
         ret = -ERANGE;
         pr_err("Count differs\n");
         goto out;
     }
 
     ret = 0;
 out:
     irq_timings_free(irq);
 
     return ret;
 }
 
 static int __init irq_timings_irqs_selftest(void)
 {
     int i, ret;
 
     for (i = 0; i < ARRAY_SIZE(tis); i++) {
         pr_info("---> Injecting intervals number #%d (count=%zd)\n",
             i, tis[i].count);
         ret = irq_timings_test_irqs(&tis[i]);
         if (ret)
             break;
     }
 
     return ret;
 }
 
 static int __init irq_timings_test_irqts(struct irq_timings *irqts,
                      unsigned count)
 {
     int start = count >= IRQ_TIMINGS_SIZE ? count - IRQ_TIMINGS_SIZE : 0;
     int i, irq, oirq = 0xBEEF;
     u64 ots = 0xDEAD, ts;
 
     /*
      * Fill the circular buffer by using the dedicated function.
      */
     for (i = 0; i < count; i++) {
         pr_debug("%d: index=%d, ts=%llX irq=%X\n",
              i, i & IRQ_TIMINGS_MASK, ots + i, oirq + i);
 
         irq_timings_push(ots + i, oirq + i);
     }
 
     /*
      * Compute the first elements values after the index wrapped
      * up or not.
      */
     ots += start;
     oirq += start;
 
     /*
      * Test the circular buffer count is correct.
      */
     pr_debug("---> Checking timings array count (%d) is right\n", count);
     if (WARN_ON(irqts->count != count))
         return -EINVAL;
 
     /*
      * Test the macro allowing to browse all the irqts.
      */
     pr_debug("---> Checking the for_each_irqts() macro\n");
     for_each_irqts(i, irqts) {
 
         irq = irq_timing_decode(irqts->values[i], &ts);
 
         pr_debug("index=%d, ts=%llX / %llX, irq=%X / %X\n",
              i, ts, ots, irq, oirq);
 
         if (WARN_ON(ts != ots || irq != oirq))
             return -EINVAL;
 
         ots++; oirq++;
     }
 
     /*
      * The circular buffer should have be flushed when browsed
      * with for_each_irqts
      */
     pr_debug("---> Checking timings array is empty after browsing it\n");
     if (WARN_ON(irqts->count))
         return -EINVAL;
 
     return 0;
 }
 
 static int __init irq_timings_irqts_selftest(void)
 {
     struct irq_timings *irqts = this_cpu_ptr(&irq_timings);
     int i, ret;
 
     /*
      * Test the circular buffer with different number of
      * elements. The purpose is to test at the limits (empty, half
      * full, full, wrapped with the cursor at the boundaries,
      * wrapped several times, etc ...
      */
     int count[] = { 0,
             IRQ_TIMINGS_SIZE >> 1,
             IRQ_TIMINGS_SIZE,
             IRQ_TIMINGS_SIZE + (IRQ_TIMINGS_SIZE >> 1),
             2 * IRQ_TIMINGS_SIZE,
             (2 * IRQ_TIMINGS_SIZE) + 3,
     };
 
     for (i = 0; i < ARRAY_SIZE(count); i++) {
 
         pr_info("---> Checking the timings with %d/%d values\n",
             count[i], IRQ_TIMINGS_SIZE);
 
         ret = irq_timings_test_irqts(irqts, count[i]);
         if (ret)
             break;
     }
 
     return ret;
 }
 
 static int __init irq_timings_selftest(void)
 {
     int ret;
 
     pr_info("------------------- selftest start -----------------\n");
 
     /*
      * At this point, we don't except any subsystem to use the irq
      * timings but us, so it should not be enabled.
      */
     if (static_branch_unlikely(&irq_timing_enabled)) {
         pr_warn("irq timings already initialized, skipping selftest\n");
         return 0;
     }
 
     ret = irq_timings_irqts_selftest();
     if (ret)
         goto out;
 
     ret = irq_timings_irqs_selftest();
     if (ret)
         goto out;
 
     ret = irq_timings_next_index_selftest();
 out:
     pr_info("---------- selftest end with %s -----------\n",
         ret ? "failure" : "success");
 
     return ret;
 }
 early_initcall(irq_timings_selftest);
 #endif

This page was automatically generated by the 2.1.0 LXR engine. The LXR team