Back to home page

LXR

 
 

    


0001 /*
0002  * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra
0003  *
0004  * Provides a framework for enqueueing and running callbacks from hardirq
0005  * context. The enqueueing is NMI-safe.
0006  */
0007 
0008 #include <linux/bug.h>
0009 #include <linux/kernel.h>
0010 #include <linux/export.h>
0011 #include <linux/irq_work.h>
0012 #include <linux/percpu.h>
0013 #include <linux/hardirq.h>
0014 #include <linux/irqflags.h>
0015 #include <linux/sched.h>
0016 #include <linux/tick.h>
0017 #include <linux/cpu.h>
0018 #include <linux/notifier.h>
0019 #include <linux/smp.h>
0020 #include <asm/processor.h>
0021 
0022 
0023 static DEFINE_PER_CPU(struct llist_head, raised_list);
0024 static DEFINE_PER_CPU(struct llist_head, lazy_list);
0025 
0026 /*
0027  * Claim the entry so that no one else will poke at it.
0028  */
0029 static bool irq_work_claim(struct irq_work *work)
0030 {
0031     unsigned long flags, oflags, nflags;
0032 
0033     /*
0034      * Start with our best wish as a premise but only trust any
0035      * flag value after cmpxchg() result.
0036      */
0037     flags = work->flags & ~IRQ_WORK_PENDING;
0038     for (;;) {
0039         nflags = flags | IRQ_WORK_FLAGS;
0040         oflags = cmpxchg(&work->flags, flags, nflags);
0041         if (oflags == flags)
0042             break;
0043         if (oflags & IRQ_WORK_PENDING)
0044             return false;
0045         flags = oflags;
0046         cpu_relax();
0047     }
0048 
0049     return true;
0050 }
0051 
0052 void __weak arch_irq_work_raise(void)
0053 {
0054     /*
0055      * Lame architectures will get the timer tick callback
0056      */
0057 }
0058 
0059 #ifdef CONFIG_SMP
0060 /*
0061  * Enqueue the irq_work @work on @cpu unless it's already pending
0062  * somewhere.
0063  *
0064  * Can be re-enqueued while the callback is still in progress.
0065  */
0066 bool irq_work_queue_on(struct irq_work *work, int cpu)
0067 {
0068     /* All work should have been flushed before going offline */
0069     WARN_ON_ONCE(cpu_is_offline(cpu));
0070 
0071     /* Arch remote IPI send/receive backend aren't NMI safe */
0072     WARN_ON_ONCE(in_nmi());
0073 
0074     /* Only queue if not already pending */
0075     if (!irq_work_claim(work))
0076         return false;
0077 
0078     if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
0079         arch_send_call_function_single_ipi(cpu);
0080 
0081     return true;
0082 }
0083 EXPORT_SYMBOL_GPL(irq_work_queue_on);
0084 #endif
0085 
0086 /* Enqueue the irq work @work on the current CPU */
0087 bool irq_work_queue(struct irq_work *work)
0088 {
0089     /* Only queue if not already pending */
0090     if (!irq_work_claim(work))
0091         return false;
0092 
0093     /* Queue the entry and raise the IPI if needed. */
0094     preempt_disable();
0095 
0096     /* If the work is "lazy", handle it from next tick if any */
0097     if (work->flags & IRQ_WORK_LAZY) {
0098         if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
0099             tick_nohz_tick_stopped())
0100             arch_irq_work_raise();
0101     } else {
0102         if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
0103             arch_irq_work_raise();
0104     }
0105 
0106     preempt_enable();
0107 
0108     return true;
0109 }
0110 EXPORT_SYMBOL_GPL(irq_work_queue);
0111 
0112 bool irq_work_needs_cpu(void)
0113 {
0114     struct llist_head *raised, *lazy;
0115 
0116     raised = this_cpu_ptr(&raised_list);
0117     lazy = this_cpu_ptr(&lazy_list);
0118 
0119     if (llist_empty(raised) || arch_irq_work_has_interrupt())
0120         if (llist_empty(lazy))
0121             return false;
0122 
0123     /* All work should have been flushed before going offline */
0124     WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
0125 
0126     return true;
0127 }
0128 
0129 static void irq_work_run_list(struct llist_head *list)
0130 {
0131     unsigned long flags;
0132     struct irq_work *work;
0133     struct llist_node *llnode;
0134 
0135     BUG_ON(!irqs_disabled());
0136 
0137     if (llist_empty(list))
0138         return;
0139 
0140     llnode = llist_del_all(list);
0141     while (llnode != NULL) {
0142         work = llist_entry(llnode, struct irq_work, llnode);
0143 
0144         llnode = llist_next(llnode);
0145 
0146         /*
0147          * Clear the PENDING bit, after this point the @work
0148          * can be re-used.
0149          * Make it immediately visible so that other CPUs trying
0150          * to claim that work don't rely on us to handle their data
0151          * while we are in the middle of the func.
0152          */
0153         flags = work->flags & ~IRQ_WORK_PENDING;
0154         xchg(&work->flags, flags);
0155 
0156         work->func(work);
0157         /*
0158          * Clear the BUSY bit and return to the free state if
0159          * no-one else claimed it meanwhile.
0160          */
0161         (void)cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY);
0162     }
0163 }
0164 
0165 /*
0166  * hotplug calls this through:
0167  *  hotplug_cfd() -> flush_smp_call_function_queue()
0168  */
0169 void irq_work_run(void)
0170 {
0171     irq_work_run_list(this_cpu_ptr(&raised_list));
0172     irq_work_run_list(this_cpu_ptr(&lazy_list));
0173 }
0174 EXPORT_SYMBOL_GPL(irq_work_run);
0175 
0176 void irq_work_tick(void)
0177 {
0178     struct llist_head *raised = this_cpu_ptr(&raised_list);
0179 
0180     if (!llist_empty(raised) && !arch_irq_work_has_interrupt())
0181         irq_work_run_list(raised);
0182     irq_work_run_list(this_cpu_ptr(&lazy_list));
0183 }
0184 
0185 /*
0186  * Synchronize against the irq_work @entry, ensures the entry is not
0187  * currently in use.
0188  */
0189 void irq_work_sync(struct irq_work *work)
0190 {
0191     WARN_ON_ONCE(irqs_disabled());
0192 
0193     while (work->flags & IRQ_WORK_BUSY)
0194         cpu_relax();
0195 }
0196 EXPORT_SYMBOL_GPL(irq_work_sync);