OSCL-LXR linux-6.0.1/​drivers/​cpuidle/​coupled.c	Source navigation Diff markup Identifier search General search
	Version: linux-6.0.1 spark-3.0.0 hadoop-3.2.0-src hadoop-3.3.0-src spark-2.4.7 linux-4.15.13 hadoop-2.7.4-src Revert   Change

 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  * coupled.c - helper functions to enter the same idle state on multiple cpus
  *
  * Copyright (c) 2011 Google, Inc.
  *
  * Author: Colin Cross <ccross@android.com>
  */
 
 #include <linux/kernel.h>
 #include <linux/cpu.h>
 #include <linux/cpuidle.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 
 #include "cpuidle.h"
 
 /**
  * DOC: Coupled cpuidle states
  *
  * On some ARM SMP SoCs (OMAP4460, Tegra 2, and probably more), the
  * cpus cannot be independently powered down, either due to
  * sequencing restrictions (on Tegra 2, cpu 0 must be the last to
  * power down), or due to HW bugs (on OMAP4460, a cpu powering up
  * will corrupt the gic state unless the other cpu runs a work
  * around).  Each cpu has a power state that it can enter without
  * coordinating with the other cpu (usually Wait For Interrupt, or
  * WFI), and one or more "coupled" power states that affect blocks
  * shared between the cpus (L2 cache, interrupt controller, and
  * sometimes the whole SoC).  Entering a coupled power state must
  * be tightly controlled on both cpus.
  *
  * This file implements a solution, where each cpu will wait in the
  * WFI state until all cpus are ready to enter a coupled state, at
  * which point the coupled state function will be called on all
  * cpus at approximately the same time.
  *
  * Once all cpus are ready to enter idle, they are woken by an smp
  * cross call.  At this point, there is a chance that one of the
  * cpus will find work to do, and choose not to enter idle.  A
  * final pass is needed to guarantee that all cpus will call the
  * power state enter function at the same time.  During this pass,
  * each cpu will increment the ready counter, and continue once the
  * ready counter matches the number of online coupled cpus.  If any
  * cpu exits idle, the other cpus will decrement their counter and
  * retry.
  *
  * requested_state stores the deepest coupled idle state each cpu
  * is ready for.  It is assumed that the states are indexed from
  * shallowest (highest power, lowest exit latency) to deepest
  * (lowest power, highest exit latency).  The requested_state
  * variable is not locked.  It is only written from the cpu that
  * it stores (or by the on/offlining cpu if that cpu is offline),
  * and only read after all the cpus are ready for the coupled idle
  * state are are no longer updating it.
  *
  * Three atomic counters are used.  alive_count tracks the number
  * of cpus in the coupled set that are currently or soon will be
  * online.  waiting_count tracks the number of cpus that are in
  * the waiting loop, in the ready loop, or in the coupled idle state.
  * ready_count tracks the number of cpus that are in the ready loop
  * or in the coupled idle state.
  *
  * To use coupled cpuidle states, a cpuidle driver must:
  *
  *    Set struct cpuidle_device.coupled_cpus to the mask of all
  *    coupled cpus, usually the same as cpu_possible_mask if all cpus
  *    are part of the same cluster.  The coupled_cpus mask must be
  *    set in the struct cpuidle_device for each cpu.
  *
  *    Set struct cpuidle_device.safe_state to a state that is not a
  *    coupled state.  This is usually WFI.
  *
  *    Set CPUIDLE_FLAG_COUPLED in struct cpuidle_state.flags for each
  *    state that affects multiple cpus.
  *
  *    Provide a struct cpuidle_state.enter function for each state
  *    that affects multiple cpus.  This function is guaranteed to be
  *    called on all cpus at approximately the same time.  The driver
  *    should ensure that the cpus all abort together if any cpu tries
  *    to abort once the function is called.  The function should return
  *    with interrupts still disabled.
  */
 
 /**
  * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
  * @coupled_cpus: mask of cpus that are part of the coupled set
  * @requested_state: array of requested states for cpus in the coupled set
  * @ready_waiting_counts: combined count of cpus  in ready or waiting loops
  * @abort_barrier: synchronisation point for abort cases
  * @online_count: count of cpus that are online
  * @refcnt: reference count of cpuidle devices that are using this struct
  * @prevent: flag to prevent coupled idle while a cpu is hotplugging
  */
 struct cpuidle_coupled {
     cpumask_t coupled_cpus;
     int requested_state[NR_CPUS];
     atomic_t ready_waiting_counts;
     atomic_t abort_barrier;
     int online_count;
     int refcnt;
     int prevent;
 };
 
 #define WAITING_BITS 16
 #define MAX_WAITING_CPUS (1 << WAITING_BITS)
 #define WAITING_MASK (MAX_WAITING_CPUS - 1)
 #define READY_MASK (~WAITING_MASK)
 
 #define CPUIDLE_COUPLED_NOT_IDLE    (-1)
 
 static DEFINE_PER_CPU(call_single_data_t, cpuidle_coupled_poke_cb);
 
 /*
  * The cpuidle_coupled_poke_pending mask is used to avoid calling
  * __smp_call_function_single with the per cpu call_single_data_t struct already
  * in use.  This prevents a deadlock where two cpus are waiting for each others
  * call_single_data_t struct to be available
  */
 static cpumask_t cpuidle_coupled_poke_pending;
 
 /*
  * The cpuidle_coupled_poked mask is used to ensure that each cpu has been poked
  * once to minimize entering the ready loop with a poke pending, which would
  * require aborting and retrying.
  */
 static cpumask_t cpuidle_coupled_poked;
 
 /**
  * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
  * @dev: cpuidle_device of the calling cpu
  * @a:   atomic variable to hold the barrier
  *
  * No caller to this function will return from this function until all online
  * cpus in the same coupled group have called this function.  Once any caller
  * has returned from this function, the barrier is immediately available for
  * reuse.
  *
  * The atomic variable must be initialized to 0 before any cpu calls
  * this function, will be reset to 0 before any cpu returns from this function.
  *
  * Must only be called from within a coupled idle state handler
  * (state.enter when state.flags has CPUIDLE_FLAG_COUPLED set).
  *
  * Provides full smp barrier semantics before and after calling.
  */
 void cpuidle_coupled_parallel_barrier(struct cpuidle_device *dev, atomic_t *a)
 {
     int n = dev->coupled->online_count;
 
     smp_mb__before_atomic();
     atomic_inc(a);
 
     while (atomic_read(a) < n)
         cpu_relax();
 
     if (atomic_inc_return(a) == n * 2) {
         atomic_set(a, 0);
         return;
     }
 
     while (atomic_read(a) > n)
         cpu_relax();
 }
 
 /**
  * cpuidle_state_is_coupled - check if a state is part of a coupled set
  * @drv: struct cpuidle_driver for the platform
  * @state: index of the target state in drv->states
  *
  * Returns true if the target state is coupled with cpus besides this one
  */
 bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state)
 {
     return drv->states[state].flags & CPUIDLE_FLAG_COUPLED;
 }
 
 /**
  * cpuidle_coupled_state_verify - check if the coupled states are correctly set.
  * @drv: struct cpuidle_driver for the platform
  *
  * Returns 0 for valid state values, a negative error code otherwise:
  *  * -EINVAL if any coupled state(safe_state_index) is wrongly set.
  */
 int cpuidle_coupled_state_verify(struct cpuidle_driver *drv)
 {
     int i;
 
     for (i = drv->state_count - 1; i >= 0; i--) {
         if (cpuidle_state_is_coupled(drv, i) &&
             (drv->safe_state_index == i ||
              drv->safe_state_index < 0 ||
              drv->safe_state_index >= drv->state_count))
             return -EINVAL;
     }
 
     return 0;
 }
 
 /**
  * cpuidle_coupled_set_ready - mark a cpu as ready
  * @coupled: the struct coupled that contains the current cpu
  */
 static inline void cpuidle_coupled_set_ready(struct cpuidle_coupled *coupled)
 {
     atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
 }
 
 /**
  * cpuidle_coupled_set_not_ready - mark a cpu as not ready
  * @coupled: the struct coupled that contains the current cpu
  *
  * Decrements the ready counter, unless the ready (and thus the waiting) counter
  * is equal to the number of online cpus.  Prevents a race where one cpu
  * decrements the waiting counter and then re-increments it just before another
  * cpu has decremented its ready counter, leading to the ready counter going
  * down from the number of online cpus without going through the coupled idle
  * state.
  *
  * Returns 0 if the counter was decremented successfully, -EINVAL if the ready
  * counter was equal to the number of online cpus.
  */
 static
 inline int cpuidle_coupled_set_not_ready(struct cpuidle_coupled *coupled)
 {
     int all;
     int ret;
 
     all = coupled->online_count | (coupled->online_count << WAITING_BITS);
     ret = atomic_add_unless(&coupled->ready_waiting_counts,
         -MAX_WAITING_CPUS, all);
 
     return ret ? 0 : -EINVAL;
 }
 
 /**
  * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
  * @coupled: the struct coupled that contains the current cpu
  *
  * Returns true if all of the cpus in a coupled set are out of the ready loop.
  */
 static inline int cpuidle_coupled_no_cpus_ready(struct cpuidle_coupled *coupled)
 {
     int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
     return r == 0;
 }
 
 /**
  * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
  * @coupled: the struct coupled that contains the current cpu
  *
  * Returns true if all cpus coupled to this target state are in the ready loop
  */
 static inline bool cpuidle_coupled_cpus_ready(struct cpuidle_coupled *coupled)
 {
     int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
     return r == coupled->online_count;
 }
 
 /**
  * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
  * @coupled: the struct coupled that contains the current cpu
  *
  * Returns true if all cpus coupled to this target state are in the wait loop
  */
 static inline bool cpuidle_coupled_cpus_waiting(struct cpuidle_coupled *coupled)
 {
     int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
     return w == coupled->online_count;
 }
 
 /**
  * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
  * @coupled: the struct coupled that contains the current cpu
  *
  * Returns true if all of the cpus in a coupled set are out of the waiting loop.
  */
 static inline int cpuidle_coupled_no_cpus_waiting(struct cpuidle_coupled *coupled)
 {
     int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
     return w == 0;
 }
 
 /**
  * cpuidle_coupled_get_state - determine the deepest idle state
  * @dev: struct cpuidle_device for this cpu
  * @coupled: the struct coupled that contains the current cpu
  *
  * Returns the deepest idle state that all coupled cpus can enter
  */
 static inline int cpuidle_coupled_get_state(struct cpuidle_device *dev,
         struct cpuidle_coupled *coupled)
 {
     int i;
     int state = INT_MAX;
 
     /*
      * Read barrier ensures that read of requested_state is ordered after
      * reads of ready_count.  Matches the write barriers
      * cpuidle_set_state_waiting.
      */
     smp_rmb();
 
     for_each_cpu(i, &coupled->coupled_cpus)
         if (cpu_online(i) && coupled->requested_state[i] < state)
             state = coupled->requested_state[i];
 
     return state;
 }
 
 static void cpuidle_coupled_handle_poke(void *info)
 {
     int cpu = (unsigned long)info;
     cpumask_set_cpu(cpu, &cpuidle_coupled_poked);
     cpumask_clear_cpu(cpu, &cpuidle_coupled_poke_pending);
 }
 
 /**
  * cpuidle_coupled_poke - wake up a cpu that may be waiting
  * @cpu: target cpu
  *
  * Ensures that the target cpu exits it's waiting idle state (if it is in it)
  * and will see updates to waiting_count before it re-enters it's waiting idle
  * state.
  *
  * If cpuidle_coupled_poked_mask is already set for the target cpu, that cpu
  * either has or will soon have a pending IPI that will wake it out of idle,
  * or it is currently processing the IPI and is not in idle.
  */
 static void cpuidle_coupled_poke(int cpu)
 {
     call_single_data_t *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);
 
     if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending))
         smp_call_function_single_async(cpu, csd);
 }
 
 /**
  * cpuidle_coupled_poke_others - wake up all other cpus that may be waiting
  * @this_cpu: target cpu
  * @coupled: the struct coupled that contains the current cpu
  *
  * Calls cpuidle_coupled_poke on all other online cpus.
  */
 static void cpuidle_coupled_poke_others(int this_cpu,
         struct cpuidle_coupled *coupled)
 {
     int cpu;
 
     for_each_cpu(cpu, &coupled->coupled_cpus)
         if (cpu != this_cpu && cpu_online(cpu))
             cpuidle_coupled_poke(cpu);
 }
 
 /**
  * cpuidle_coupled_set_waiting - mark this cpu as in the wait loop
  * @cpu: target cpu
  * @coupled: the struct coupled that contains the current cpu
  * @next_state: the index in drv->states of the requested state for this cpu
  *
  * Updates the requested idle state for the specified cpuidle device.
  * Returns the number of waiting cpus.
  */
 static int cpuidle_coupled_set_waiting(int cpu,
         struct cpuidle_coupled *coupled, int next_state)
 {
     coupled->requested_state[cpu] = next_state;
 
     /*
      * The atomic_inc_return provides a write barrier to order the write
      * to requested_state with the later write that increments ready_count.
      */
     return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;
 }
 
 /**
  * cpuidle_coupled_set_not_waiting - mark this cpu as leaving the wait loop
  * @cpu: target cpu
  * @coupled: the struct coupled that contains the current cpu
  *
  * Removes the requested idle state for the specified cpuidle device.
  */
 static void cpuidle_coupled_set_not_waiting(int cpu,
         struct cpuidle_coupled *coupled)
 {
     /*
      * Decrementing waiting count can race with incrementing it in
      * cpuidle_coupled_set_waiting, but that's OK.  Worst case, some
      * cpus will increment ready_count and then spin until they
      * notice that this cpu has cleared it's requested_state.
      */
     atomic_dec(&coupled->ready_waiting_counts);
 
     coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE;
 }
 
 /**
  * cpuidle_coupled_set_done - mark this cpu as leaving the ready loop
  * @cpu: the current cpu
  * @coupled: the struct coupled that contains the current cpu
  *
  * Marks this cpu as no longer in the ready and waiting loops.  Decrements
  * the waiting count first to prevent another cpu looping back in and seeing
  * this cpu as waiting just before it exits idle.
  */
 static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled)
 {
     cpuidle_coupled_set_not_waiting(cpu, coupled);
     atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
 }
 
 /**
  * cpuidle_coupled_clear_pokes - spin until the poke interrupt is processed
  * @cpu: this cpu
  *
  * Turns on interrupts and spins until any outstanding poke interrupts have
  * been processed and the poke bit has been cleared.
  *
  * Other interrupts may also be processed while interrupts are enabled, so
  * need_resched() must be tested after this function returns to make sure
  * the interrupt didn't schedule work that should take the cpu out of idle.
  *
  * Returns 0 if no poke was pending, 1 if a poke was cleared.
  */
 static int cpuidle_coupled_clear_pokes(int cpu)
 {
     if (!cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))
         return 0;
 
     local_irq_enable();
     while (cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))
         cpu_relax();
     local_irq_disable();
 
     return 1;
 }
 
 static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled)
 {
     cpumask_t cpus;
     int ret;
 
     cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);
     ret = cpumask_and(&cpus, &cpuidle_coupled_poke_pending, &cpus);
 
     return ret;
 }
 
 /**
  * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
  * @dev: struct cpuidle_device for the current cpu
  * @drv: struct cpuidle_driver for the platform
  * @next_state: index of the requested state in drv->states
  *
  * Coordinate with coupled cpus to enter the target state.  This is a two
  * stage process.  In the first stage, the cpus are operating independently,
  * and may call into cpuidle_enter_state_coupled at completely different times.
  * To save as much power as possible, the first cpus to call this function will
  * go to an intermediate state (the cpuidle_device's safe state), and wait for
  * all the other cpus to call this function.  Once all coupled cpus are idle,
  * the second stage will start.  Each coupled cpu will spin until all cpus have
  * guaranteed that they will call the target_state.
  *
  * This function must be called with interrupts disabled.  It may enable
  * interrupts while preparing for idle, and it will always return with
  * interrupts enabled.
  */
 int cpuidle_enter_state_coupled(struct cpuidle_device *dev,
         struct cpuidle_driver *drv, int next_state)
 {
     int entered_state = -1;
     struct cpuidle_coupled *coupled = dev->coupled;
     int w;
 
     if (!coupled)
         return -EINVAL;
 
     while (coupled->prevent) {
         cpuidle_coupled_clear_pokes(dev->cpu);
         if (need_resched()) {
             local_irq_enable();
             return entered_state;
         }
         entered_state = cpuidle_enter_state(dev, drv,
             drv->safe_state_index);
         local_irq_disable();
     }
 
     /* Read barrier ensures online_count is read after prevent is cleared */
     smp_rmb();
 
 reset:
     cpumask_clear_cpu(dev->cpu, &cpuidle_coupled_poked);
 
     w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);
     /*
      * If this is the last cpu to enter the waiting state, poke
      * all the other cpus out of their waiting state so they can
      * enter a deeper state.  This can race with one of the cpus
      * exiting the waiting state due to an interrupt and
      * decrementing waiting_count, see comment below.
      */
     if (w == coupled->online_count) {
         cpumask_set_cpu(dev->cpu, &cpuidle_coupled_poked);
         cpuidle_coupled_poke_others(dev->cpu, coupled);
     }
 
 retry:
     /*
      * Wait for all coupled cpus to be idle, using the deepest state
      * allowed for a single cpu.  If this was not the poking cpu, wait
      * for at least one poke before leaving to avoid a race where
      * two cpus could arrive at the waiting loop at the same time,
      * but the first of the two to arrive could skip the loop without
      * processing the pokes from the last to arrive.
      */
     while (!cpuidle_coupled_cpus_waiting(coupled) ||
             !cpumask_test_cpu(dev->cpu, &cpuidle_coupled_poked)) {
         if (cpuidle_coupled_clear_pokes(dev->cpu))
             continue;
 
         if (need_resched()) {
             cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
             goto out;
         }
 
         if (coupled->prevent) {
             cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
             goto out;
         }
 
         entered_state = cpuidle_enter_state(dev, drv,
             drv->safe_state_index);
         local_irq_disable();
     }
 
     cpuidle_coupled_clear_pokes(dev->cpu);
     if (need_resched()) {
         cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
         goto out;
     }
 
     /*
      * Make sure final poke status for this cpu is visible before setting
      * cpu as ready.
      */
     smp_wmb();
 
     /*
      * All coupled cpus are probably idle.  There is a small chance that
      * one of the other cpus just became active.  Increment the ready count,
      * and spin until all coupled cpus have incremented the counter. Once a
      * cpu has incremented the ready counter, it cannot abort idle and must
      * spin until either all cpus have incremented the ready counter, or
      * another cpu leaves idle and decrements the waiting counter.
      */
 
     cpuidle_coupled_set_ready(coupled);
     while (!cpuidle_coupled_cpus_ready(coupled)) {
         /* Check if any other cpus bailed out of idle. */
         if (!cpuidle_coupled_cpus_waiting(coupled))
             if (!cpuidle_coupled_set_not_ready(coupled))
                 goto retry;
 
         cpu_relax();
     }
 
     /*
      * Make sure read of all cpus ready is done before reading pending pokes
      */
     smp_rmb();
 
     /*
      * There is a small chance that a cpu left and reentered idle after this
      * cpu saw that all cpus were waiting.  The cpu that reentered idle will
      * have sent this cpu a poke, which will still be pending after the
      * ready loop.  The pending interrupt may be lost by the interrupt
      * controller when entering the deep idle state.  It's not possible to
      * clear a pending interrupt without turning interrupts on and handling
      * it, and it's too late to turn on interrupts here, so reset the
      * coupled idle state of all cpus and retry.
      */
     if (cpuidle_coupled_any_pokes_pending(coupled)) {
         cpuidle_coupled_set_done(dev->cpu, coupled);
         /* Wait for all cpus to see the pending pokes */
         cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier);
         goto reset;
     }
 
     /* all cpus have acked the coupled state */
     next_state = cpuidle_coupled_get_state(dev, coupled);
 
     entered_state = cpuidle_enter_state(dev, drv, next_state);
 
     cpuidle_coupled_set_done(dev->cpu, coupled);
 
 out:
     /*
      * Normal cpuidle states are expected to return with irqs enabled.
      * That leads to an inefficiency where a cpu receiving an interrupt
      * that brings it out of idle will process that interrupt before
      * exiting the idle enter function and decrementing ready_count.  All
      * other cpus will need to spin waiting for the cpu that is processing
      * the interrupt.  If the driver returns with interrupts disabled,
      * all other cpus will loop back into the safe idle state instead of
      * spinning, saving power.
      *
      * Calling local_irq_enable here allows coupled states to return with
      * interrupts disabled, but won't cause problems for drivers that
      * exit with interrupts enabled.
      */
     local_irq_enable();
 
     /*
      * Wait until all coupled cpus have exited idle.  There is no risk that
      * a cpu exits and re-enters the ready state because this cpu has
      * already decremented its waiting_count.
      */
     while (!cpuidle_coupled_no_cpus_ready(coupled))
         cpu_relax();
 
     return entered_state;
 }
 
 static void cpuidle_coupled_update_online_cpus(struct cpuidle_coupled *coupled)
 {
     cpumask_t cpus;
     cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);
     coupled->online_count = cpumask_weight(&cpus);
 }
 
 /**
  * cpuidle_coupled_register_device - register a coupled cpuidle device
  * @dev: struct cpuidle_device for the current cpu
  *
  * Called from cpuidle_register_device to handle coupled idle init.  Finds the
  * cpuidle_coupled struct for this set of coupled cpus, or creates one if none
  * exists yet.
  */
 int cpuidle_coupled_register_device(struct cpuidle_device *dev)
 {
     int cpu;
     struct cpuidle_device *other_dev;
     call_single_data_t *csd;
     struct cpuidle_coupled *coupled;
 
     if (cpumask_empty(&dev->coupled_cpus))
         return 0;
 
     for_each_cpu(cpu, &dev->coupled_cpus) {
         other_dev = per_cpu(cpuidle_devices, cpu);
         if (other_dev && other_dev->coupled) {
             coupled = other_dev->coupled;
             goto have_coupled;
         }
     }
 
     /* No existing coupled info found, create a new one */
     coupled = kzalloc(sizeof(struct cpuidle_coupled), GFP_KERNEL);
     if (!coupled)
         return -ENOMEM;
 
     coupled->coupled_cpus = dev->coupled_cpus;
 
 have_coupled:
     dev->coupled = coupled;
     if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus)))
         coupled->prevent++;
 
     cpuidle_coupled_update_online_cpus(coupled);
 
     coupled->refcnt++;
 
     csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu);
     INIT_CSD(csd, cpuidle_coupled_handle_poke, (void *)(unsigned long)dev->cpu);
 
     return 0;
 }
 
 /**
  * cpuidle_coupled_unregister_device - unregister a coupled cpuidle device
  * @dev: struct cpuidle_device for the current cpu
  *
  * Called from cpuidle_unregister_device to tear down coupled idle.  Removes the
  * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if
  * this was the last cpu in the set.
  */
 void cpuidle_coupled_unregister_device(struct cpuidle_device *dev)
 {
     struct cpuidle_coupled *coupled = dev->coupled;
 
     if (cpumask_empty(&dev->coupled_cpus))
         return;
 
     if (--coupled->refcnt)
         kfree(coupled);
     dev->coupled = NULL;
 }
 
 /**
  * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
  * @coupled: the struct coupled that contains the cpu that is changing state
  *
  * Disables coupled cpuidle on a coupled set of cpus.  Used to ensure that
  * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
  */
 static void cpuidle_coupled_prevent_idle(struct cpuidle_coupled *coupled)
 {
     int cpu = get_cpu();
 
     /* Force all cpus out of the waiting loop. */
     coupled->prevent++;
     cpuidle_coupled_poke_others(cpu, coupled);
     put_cpu();
     while (!cpuidle_coupled_no_cpus_waiting(coupled))
         cpu_relax();
 }
 
 /**
  * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
  * @coupled: the struct coupled that contains the cpu that is changing state
  *
  * Enables coupled cpuidle on a coupled set of cpus.  Used to ensure that
  * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
  */
 static void cpuidle_coupled_allow_idle(struct cpuidle_coupled *coupled)
 {
     int cpu = get_cpu();
 
     /*
      * Write barrier ensures readers see the new online_count when they
      * see prevent == 0.
      */
     smp_wmb();
     coupled->prevent--;
     /* Force cpus out of the prevent loop. */
     cpuidle_coupled_poke_others(cpu, coupled);
     put_cpu();
 }
 
 static int coupled_cpu_online(unsigned int cpu)
 {
     struct cpuidle_device *dev;
 
     mutex_lock(&cpuidle_lock);
 
     dev = per_cpu(cpuidle_devices, cpu);
     if (dev && dev->coupled) {
         cpuidle_coupled_update_online_cpus(dev->coupled);
         cpuidle_coupled_allow_idle(dev->coupled);
     }
 
     mutex_unlock(&cpuidle_lock);
     return 0;
 }
 
 static int coupled_cpu_up_prepare(unsigned int cpu)
 {
     struct cpuidle_device *dev;
 
     mutex_lock(&cpuidle_lock);
 
     dev = per_cpu(cpuidle_devices, cpu);
     if (dev && dev->coupled)
         cpuidle_coupled_prevent_idle(dev->coupled);
 
     mutex_unlock(&cpuidle_lock);
     return 0;
 }
 
 static int __init cpuidle_coupled_init(void)
 {
     int ret;
 
     ret = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE,
                     "cpuidle/coupled:prepare",
                     coupled_cpu_up_prepare,
                     coupled_cpu_online);
     if (ret)
         return ret;
     ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
                     "cpuidle/coupled:online",
                     coupled_cpu_online,
                     coupled_cpu_up_prepare);
     if (ret < 0)
         cpuhp_remove_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE);
     return ret;
 }
 core_initcall(cpuidle_coupled_init);

This page was automatically generated by the 2.1.0 LXR engine. The LXR team