Back to home page

OSCL-LXR
 
 

     

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * IRQ subsystem internal functions and variables:
0004  *
0005  * Do not ever include this file from anything else than
0006  * kernel/irq/. Do not even think about using any information outside
0007  * of this file for your non core code.
0008  */
0009 #include <linux/irqdesc.h>
0010 #include <linux/kernel_stat.h>
0011 #include <linux/pm_runtime.h>
0012 #include <linux/sched/clock.h>
0013 
0014 #ifdef CONFIG_SPARSE_IRQ
0015 # define IRQ_BITMAP_BITS    (NR_IRQS + 8196)
0016 #else
0017 # define IRQ_BITMAP_BITS    NR_IRQS
0018 #endif
0019 
0020 #define istate core_internal_state__do_not_mess_with_it
0021 
0022 extern bool noirqdebug;
0023 
0024 extern struct irqaction chained_action;
0025 
0026 /*
0027  * Bits used by threaded handlers:
0028  * IRQTF_RUNTHREAD - signals that the interrupt handler thread should run
0029  * IRQTF_WARNED    - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed
0030  * IRQTF_AFFINITY  - irq thread is requested to adjust affinity
0031  * IRQTF_FORCED_THREAD  - irq action is force threaded
0032  * IRQTF_READY     - signals that irq thread is ready
0033  */
0034 enum {
0035     IRQTF_RUNTHREAD,
0036     IRQTF_WARNED,
0037     IRQTF_AFFINITY,
0038     IRQTF_FORCED_THREAD,
0039     IRQTF_READY,
0040 };
0041 
0042 /*
0043  * Bit masks for desc->core_internal_state__do_not_mess_with_it
0044  *
0045  * IRQS_AUTODETECT      - autodetection in progress
0046  * IRQS_SPURIOUS_DISABLED   - was disabled due to spurious interrupt
0047  *                detection
0048  * IRQS_POLL_INPROGRESS     - polling in progress
0049  * IRQS_ONESHOT         - irq is not unmasked in primary handler
0050  * IRQS_REPLAY          - irq is replayed
0051  * IRQS_WAITING         - irq is waiting
0052  * IRQS_PENDING         - irq is pending and replayed later
0053  * IRQS_SUSPENDED       - irq is suspended
0054  * IRQS_NMI         - irq line is used to deliver NMIs
0055  */
0056 enum {
0057     IRQS_AUTODETECT     = 0x00000001,
0058     IRQS_SPURIOUS_DISABLED  = 0x00000002,
0059     IRQS_POLL_INPROGRESS    = 0x00000008,
0060     IRQS_ONESHOT        = 0x00000020,
0061     IRQS_REPLAY     = 0x00000040,
0062     IRQS_WAITING        = 0x00000080,
0063     IRQS_PENDING        = 0x00000200,
0064     IRQS_SUSPENDED      = 0x00000800,
0065     IRQS_TIMINGS        = 0x00001000,
0066     IRQS_NMI        = 0x00002000,
0067 };
0068 
0069 #include "debug.h"
0070 #include "settings.h"
0071 
0072 extern int __irq_set_trigger(struct irq_desc *desc, unsigned long flags);
0073 extern void __disable_irq(struct irq_desc *desc);
0074 extern void __enable_irq(struct irq_desc *desc);
0075 
0076 #define IRQ_RESEND  true
0077 #define IRQ_NORESEND    false
0078 
0079 #define IRQ_START_FORCE true
0080 #define IRQ_START_COND  false
0081 
0082 extern int irq_activate(struct irq_desc *desc);
0083 extern int irq_activate_and_startup(struct irq_desc *desc, bool resend);
0084 extern int irq_startup(struct irq_desc *desc, bool resend, bool force);
0085 
0086 extern void irq_shutdown(struct irq_desc *desc);
0087 extern void irq_shutdown_and_deactivate(struct irq_desc *desc);
0088 extern void irq_enable(struct irq_desc *desc);
0089 extern void irq_disable(struct irq_desc *desc);
0090 extern void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu);
0091 extern void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu);
0092 extern void mask_irq(struct irq_desc *desc);
0093 extern void unmask_irq(struct irq_desc *desc);
0094 extern void unmask_threaded_irq(struct irq_desc *desc);
0095 
0096 #ifdef CONFIG_SPARSE_IRQ
0097 static inline void irq_mark_irq(unsigned int irq) { }
0098 #else
0099 extern void irq_mark_irq(unsigned int irq);
0100 #endif
0101 
0102 extern int __irq_get_irqchip_state(struct irq_data *data,
0103                    enum irqchip_irq_state which,
0104                    bool *state);
0105 
0106 extern void init_kstat_irqs(struct irq_desc *desc, int node, int nr);
0107 
0108 irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc);
0109 irqreturn_t handle_irq_event_percpu(struct irq_desc *desc);
0110 irqreturn_t handle_irq_event(struct irq_desc *desc);
0111 
0112 /* Resending of interrupts :*/
0113 int check_irq_resend(struct irq_desc *desc, bool inject);
0114 bool irq_wait_for_poll(struct irq_desc *desc);
0115 void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action);
0116 
0117 #ifdef CONFIG_PROC_FS
0118 extern void register_irq_proc(unsigned int irq, struct irq_desc *desc);
0119 extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc);
0120 extern void register_handler_proc(unsigned int irq, struct irqaction *action);
0121 extern void unregister_handler_proc(unsigned int irq, struct irqaction *action);
0122 #else
0123 static inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { }
0124 static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { }
0125 static inline void register_handler_proc(unsigned int irq,
0126                      struct irqaction *action) { }
0127 static inline void unregister_handler_proc(unsigned int irq,
0128                        struct irqaction *action) { }
0129 #endif
0130 
0131 extern bool irq_can_set_affinity_usr(unsigned int irq);
0132 
0133 extern void irq_set_thread_affinity(struct irq_desc *desc);
0134 
0135 extern int irq_do_set_affinity(struct irq_data *data,
0136                    const struct cpumask *dest, bool force);
0137 
0138 #ifdef CONFIG_SMP
0139 extern int irq_setup_affinity(struct irq_desc *desc);
0140 #else
0141 static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; }
0142 #endif
0143 
0144 /* Inline functions for support of irq chips on slow busses */
0145 static inline void chip_bus_lock(struct irq_desc *desc)
0146 {
0147     if (unlikely(desc->irq_data.chip->irq_bus_lock))
0148         desc->irq_data.chip->irq_bus_lock(&desc->irq_data);
0149 }
0150 
0151 static inline void chip_bus_sync_unlock(struct irq_desc *desc)
0152 {
0153     if (unlikely(desc->irq_data.chip->irq_bus_sync_unlock))
0154         desc->irq_data.chip->irq_bus_sync_unlock(&desc->irq_data);
0155 }
0156 
0157 #define _IRQ_DESC_CHECK     (1 << 0)
0158 #define _IRQ_DESC_PERCPU    (1 << 1)
0159 
0160 #define IRQ_GET_DESC_CHECK_GLOBAL   (_IRQ_DESC_CHECK)
0161 #define IRQ_GET_DESC_CHECK_PERCPU   (_IRQ_DESC_CHECK | _IRQ_DESC_PERCPU)
0162 
0163 #define for_each_action_of_desc(desc, act)          \
0164     for (act = desc->action; act; act = act->next)
0165 
0166 struct irq_desc *
0167 __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
0168             unsigned int check);
0169 void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus);
0170 
0171 static inline struct irq_desc *
0172 irq_get_desc_buslock(unsigned int irq, unsigned long *flags, unsigned int check)
0173 {
0174     return __irq_get_desc_lock(irq, flags, true, check);
0175 }
0176 
0177 static inline void
0178 irq_put_desc_busunlock(struct irq_desc *desc, unsigned long flags)
0179 {
0180     __irq_put_desc_unlock(desc, flags, true);
0181 }
0182 
0183 static inline struct irq_desc *
0184 irq_get_desc_lock(unsigned int irq, unsigned long *flags, unsigned int check)
0185 {
0186     return __irq_get_desc_lock(irq, flags, false, check);
0187 }
0188 
0189 static inline void
0190 irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags)
0191 {
0192     __irq_put_desc_unlock(desc, flags, false);
0193 }
0194 
0195 #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors)
0196 
0197 static inline unsigned int irqd_get(struct irq_data *d)
0198 {
0199     return __irqd_to_state(d);
0200 }
0201 
0202 /*
0203  * Manipulation functions for irq_data.state
0204  */
0205 static inline void irqd_set_move_pending(struct irq_data *d)
0206 {
0207     __irqd_to_state(d) |= IRQD_SETAFFINITY_PENDING;
0208 }
0209 
0210 static inline void irqd_clr_move_pending(struct irq_data *d)
0211 {
0212     __irqd_to_state(d) &= ~IRQD_SETAFFINITY_PENDING;
0213 }
0214 
0215 static inline void irqd_set_managed_shutdown(struct irq_data *d)
0216 {
0217     __irqd_to_state(d) |= IRQD_MANAGED_SHUTDOWN;
0218 }
0219 
0220 static inline void irqd_clr_managed_shutdown(struct irq_data *d)
0221 {
0222     __irqd_to_state(d) &= ~IRQD_MANAGED_SHUTDOWN;
0223 }
0224 
0225 static inline void irqd_clear(struct irq_data *d, unsigned int mask)
0226 {
0227     __irqd_to_state(d) &= ~mask;
0228 }
0229 
0230 static inline void irqd_set(struct irq_data *d, unsigned int mask)
0231 {
0232     __irqd_to_state(d) |= mask;
0233 }
0234 
0235 static inline bool irqd_has_set(struct irq_data *d, unsigned int mask)
0236 {
0237     return __irqd_to_state(d) & mask;
0238 }
0239 
0240 static inline void irq_state_set_disabled(struct irq_desc *desc)
0241 {
0242     irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
0243 }
0244 
0245 static inline void irq_state_set_masked(struct irq_desc *desc)
0246 {
0247     irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
0248 }
0249 
0250 #undef __irqd_to_state
0251 
0252 static inline void __kstat_incr_irqs_this_cpu(struct irq_desc *desc)
0253 {
0254     __this_cpu_inc(*desc->kstat_irqs);
0255     __this_cpu_inc(kstat.irqs_sum);
0256 }
0257 
0258 static inline void kstat_incr_irqs_this_cpu(struct irq_desc *desc)
0259 {
0260     __kstat_incr_irqs_this_cpu(desc);
0261     desc->tot_count++;
0262 }
0263 
0264 static inline int irq_desc_get_node(struct irq_desc *desc)
0265 {
0266     return irq_common_data_get_node(&desc->irq_common_data);
0267 }
0268 
0269 static inline int irq_desc_is_chained(struct irq_desc *desc)
0270 {
0271     return (desc->action && desc->action == &chained_action);
0272 }
0273 
0274 #ifdef CONFIG_PM_SLEEP
0275 bool irq_pm_check_wakeup(struct irq_desc *desc);
0276 void irq_pm_install_action(struct irq_desc *desc, struct irqaction *action);
0277 void irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action);
0278 #else
0279 static inline bool irq_pm_check_wakeup(struct irq_desc *desc) { return false; }
0280 static inline void
0281 irq_pm_install_action(struct irq_desc *desc, struct irqaction *action) { }
0282 static inline void
0283 irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { }
0284 #endif
0285 
0286 #ifdef CONFIG_IRQ_TIMINGS
0287 
0288 #define IRQ_TIMINGS_SHIFT   5
0289 #define IRQ_TIMINGS_SIZE    (1 << IRQ_TIMINGS_SHIFT)
0290 #define IRQ_TIMINGS_MASK    (IRQ_TIMINGS_SIZE - 1)
0291 
0292 /**
0293  * struct irq_timings - irq timings storing structure
0294  * @values: a circular buffer of u64 encoded <timestamp,irq> values
0295  * @count: the number of elements in the array
0296  */
0297 struct irq_timings {
0298     u64 values[IRQ_TIMINGS_SIZE];
0299     int count;
0300 };
0301 
0302 DECLARE_PER_CPU(struct irq_timings, irq_timings);
0303 
0304 extern void irq_timings_free(int irq);
0305 extern int irq_timings_alloc(int irq);
0306 
0307 static inline void irq_remove_timings(struct irq_desc *desc)
0308 {
0309     desc->istate &= ~IRQS_TIMINGS;
0310 
0311     irq_timings_free(irq_desc_get_irq(desc));
0312 }
0313 
0314 static inline void irq_setup_timings(struct irq_desc *desc, struct irqaction *act)
0315 {
0316     int irq = irq_desc_get_irq(desc);
0317     int ret;
0318 
0319     /*
0320      * We don't need the measurement because the idle code already
0321      * knows the next expiry event.
0322      */
0323     if (act->flags & __IRQF_TIMER)
0324         return;
0325 
0326     /*
0327      * In case the timing allocation fails, we just want to warn,
0328      * not fail, so letting the system boot anyway.
0329      */
0330     ret = irq_timings_alloc(irq);
0331     if (ret) {
0332         pr_warn("Failed to allocate irq timing stats for irq%d (%d)",
0333             irq, ret);
0334         return;
0335     }
0336 
0337     desc->istate |= IRQS_TIMINGS;
0338 }
0339 
0340 extern void irq_timings_enable(void);
0341 extern void irq_timings_disable(void);
0342 
0343 DECLARE_STATIC_KEY_FALSE(irq_timing_enabled);
0344 
0345 /*
0346  * The interrupt number and the timestamp are encoded into a single
0347  * u64 variable to optimize the size.
0348  * 48 bit time stamp and 16 bit IRQ number is way sufficient.
0349  *  Who cares an IRQ after 78 hours of idle time?
0350  */
0351 static inline u64 irq_timing_encode(u64 timestamp, int irq)
0352 {
0353     return (timestamp << 16) | irq;
0354 }
0355 
0356 static inline int irq_timing_decode(u64 value, u64 *timestamp)
0357 {
0358     *timestamp = value >> 16;
0359     return value & U16_MAX;
0360 }
0361 
0362 static __always_inline void irq_timings_push(u64 ts, int irq)
0363 {
0364     struct irq_timings *timings = this_cpu_ptr(&irq_timings);
0365 
0366     timings->values[timings->count & IRQ_TIMINGS_MASK] =
0367         irq_timing_encode(ts, irq);
0368 
0369     timings->count++;
0370 }
0371 
0372 /*
0373  * The function record_irq_time is only called in one place in the
0374  * interrupts handler. We want this function always inline so the code
0375  * inside is embedded in the function and the static key branching
0376  * code can act at the higher level. Without the explicit
0377  * __always_inline we can end up with a function call and a small
0378  * overhead in the hotpath for nothing.
0379  */
0380 static __always_inline void record_irq_time(struct irq_desc *desc)
0381 {
0382     if (!static_branch_likely(&irq_timing_enabled))
0383         return;
0384 
0385     if (desc->istate & IRQS_TIMINGS)
0386         irq_timings_push(local_clock(), irq_desc_get_irq(desc));
0387 }
0388 #else
0389 static inline void irq_remove_timings(struct irq_desc *desc) {}
0390 static inline void irq_setup_timings(struct irq_desc *desc,
0391                      struct irqaction *act) {};
0392 static inline void record_irq_time(struct irq_desc *desc) {}
0393 #endif /* CONFIG_IRQ_TIMINGS */
0394 
0395 
0396 #ifdef CONFIG_GENERIC_IRQ_CHIP
0397 void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
0398                int num_ct, unsigned int irq_base,
0399                void __iomem *reg_base, irq_flow_handler_t handler);
0400 #else
0401 static inline void
0402 irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
0403               int num_ct, unsigned int irq_base,
0404               void __iomem *reg_base, irq_flow_handler_t handler) { }
0405 #endif /* CONFIG_GENERIC_IRQ_CHIP */
0406 
0407 #ifdef CONFIG_GENERIC_PENDING_IRQ
0408 static inline bool irq_can_move_pcntxt(struct irq_data *data)
0409 {
0410     return irqd_can_move_in_process_context(data);
0411 }
0412 static inline bool irq_move_pending(struct irq_data *data)
0413 {
0414     return irqd_is_setaffinity_pending(data);
0415 }
0416 static inline void
0417 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
0418 {
0419     cpumask_copy(desc->pending_mask, mask);
0420 }
0421 static inline void
0422 irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
0423 {
0424     cpumask_copy(mask, desc->pending_mask);
0425 }
0426 static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
0427 {
0428     return desc->pending_mask;
0429 }
0430 static inline bool handle_enforce_irqctx(struct irq_data *data)
0431 {
0432     return irqd_is_handle_enforce_irqctx(data);
0433 }
0434 bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear);
0435 #else /* CONFIG_GENERIC_PENDING_IRQ */
0436 static inline bool irq_can_move_pcntxt(struct irq_data *data)
0437 {
0438     return true;
0439 }
0440 static inline bool irq_move_pending(struct irq_data *data)
0441 {
0442     return false;
0443 }
0444 static inline void
0445 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
0446 {
0447 }
0448 static inline void
0449 irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
0450 {
0451 }
0452 static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
0453 {
0454     return NULL;
0455 }
0456 static inline bool irq_fixup_move_pending(struct irq_desc *desc, bool fclear)
0457 {
0458     return false;
0459 }
0460 static inline bool handle_enforce_irqctx(struct irq_data *data)
0461 {
0462     return false;
0463 }
0464 #endif /* !CONFIG_GENERIC_PENDING_IRQ */
0465 
0466 #if !defined(CONFIG_IRQ_DOMAIN) || !defined(CONFIG_IRQ_DOMAIN_HIERARCHY)
0467 static inline int irq_domain_activate_irq(struct irq_data *data, bool reserve)
0468 {
0469     irqd_set_activated(data);
0470     return 0;
0471 }
0472 static inline void irq_domain_deactivate_irq(struct irq_data *data)
0473 {
0474     irqd_clr_activated(data);
0475 }
0476 #endif
0477 
0478 static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd)
0479 {
0480 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
0481     return irqd->parent_data;
0482 #else
0483     return NULL;
0484 #endif
0485 }
0486 
0487 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS
0488 #include <linux/debugfs.h>
0489 
0490 void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc);
0491 static inline void irq_remove_debugfs_entry(struct irq_desc *desc)
0492 {
0493     debugfs_remove(desc->debugfs_file);
0494     kfree(desc->dev_name);
0495 }
0496 void irq_debugfs_copy_devname(int irq, struct device *dev);
0497 # ifdef CONFIG_IRQ_DOMAIN
0498 void irq_domain_debugfs_init(struct dentry *root);
0499 # else
0500 static inline void irq_domain_debugfs_init(struct dentry *root)
0501 {
0502 }
0503 # endif
0504 #else /* CONFIG_GENERIC_IRQ_DEBUGFS */
0505 static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d)
0506 {
0507 }
0508 static inline void irq_remove_debugfs_entry(struct irq_desc *d)
0509 {
0510 }
0511 static inline void irq_debugfs_copy_devname(int irq, struct device *dev)
0512 {
0513 }
0514 #endif /* CONFIG_GENERIC_IRQ_DEBUGFS */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!