1 /*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Fixed a disable_bh()/enable_bh() race (was causing a console lockup)
7 * due bh_mask_count not atomic handling. Copyright (C) 1998 Andrea Arcangeli
8 *
9 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
10 */
11
12 #include <linux/config.h>
13 #include <linux/mm.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/smp_lock.h>
17 #include <linux/init.h>
18 #include <linux/tqueue.h>
19
20 /*
21 - No shared variables, all the data are CPU local.
22 - If a softirq needs serialization, let it serialize itself
23 by its own spinlocks.
24 - Even if softirq is serialized, only local cpu is marked for
25 execution. Hence, we get something sort of weak cpu binding.
26 Though it is still not clear, will it result in better locality
27 or will not.
28 - These softirqs are not masked by global cli() and start_bh_atomic()
29 (by clear reasons). Hence, old parts of code still using global locks
30 MUST NOT use softirqs, but insert interfacing routines acquiring
31 global locks. F.e. look at BHs implementation.
32
33 Examples:
34 - NET RX softirq. It is multithreaded and does not require
35 any global serialization.
36 - NET TX softirq. It kicks software netdevice queues, hence
37 it is logically serialized per device, but this serialization
38 is invisible to common code.
39 - Tasklets: serialized wrt itself.
40 - Bottom halves: globally serialized, grr...
41 */
42
43 /* No separate irq_stat for s390, it is part of PSA */
44 #if !defined(CONFIG_ARCH_S390)
45 irq_cpustat_t irq_stat[NR_CPUS];
46 #endif /* CONFIG_ARCH_S390 */
47
48 static struct softirq_action softirq_vec[32] __cacheline_aligned;
49
50 asmlinkage void do_softirq()
51 {
52 int cpu = smp_processor_id();
53 __u32 active, mask;
54
55 if (in_interrupt())
56 return;
57
58 local_bh_disable();
59
60 local_irq_disable();
61 mask = softirq_mask(cpu);
62 active = softirq_active(cpu) & mask;
63
64 if (active) {
65 struct softirq_action *h;
66
67 restart:
68 /* Reset active bitmask before enabling irqs */
69 softirq_active(cpu) &= ~active;
70
71 local_irq_enable();
72
73 h = softirq_vec;
74 mask &= ~active;
75
76 do {
77 if (active & 1)
78 h->action(h);
79 h++;
80 active >>= 1;
81 } while (active);
82
83 local_irq_disable();
84
85 active = softirq_active(cpu);
86 if ((active &= mask) != 0)
87 goto retry;
88 }
89
90 local_bh_enable();
91
92 /* Leave with locally disabled hard irqs. It is critical to close
93 * window for infinite recursion, while we help local bh count,
94 * it protected us. Now we are defenceless.
95 */
96 return;
97
98 retry:
99 goto restart;
100 }
101
102
103 static spinlock_t softirq_mask_lock = SPIN_LOCK_UNLOCKED;
104
105 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
106 {
107 unsigned long flags;
108 int i;
109
110 spin_lock_irqsave(&softirq_mask_lock, flags);
111 softirq_vec[nr].data = data;
112 softirq_vec[nr].action = action;
113
114 for (i=0; i<NR_CPUS; i++)
115 softirq_mask(i) |= (1<<nr);
116 spin_unlock_irqrestore(&softirq_mask_lock, flags);
117 }
118
119
120 /* Tasklets */
121
122 struct tasklet_head tasklet_vec[NR_CPUS] __cacheline_aligned;
123
124 static void tasklet_action(struct softirq_action *a)
125 {
126 int cpu = smp_processor_id();
127 struct tasklet_struct *list;
128
129 local_irq_disable();
130 list = tasklet_vec[cpu].list;
131 tasklet_vec[cpu].list = NULL;
132 local_irq_enable();
133
134 while (list != NULL) {
135 struct tasklet_struct *t = list;
136
137 list = list->next;
138
139 if (tasklet_trylock(t)) {
140 if (atomic_read(&t->count) == 0) {
141 clear_bit(TASKLET_STATE_SCHED, &t->state);
142
143 t->func(t->data);
144 /*
145 * talklet_trylock() uses test_and_set_bit that imply
146 * an mb when it returns zero, thus we need the explicit
147 * mb only here: while closing the critical section.
148 */
149 #ifdef CONFIG_SMP
150 smp_mb__before_clear_bit();
151 #endif
152 tasklet_unlock(t);
153 continue;
154 }
155 tasklet_unlock(t);
156 }
157 local_irq_disable();
158 t->next = tasklet_vec[cpu].list;
159 tasklet_vec[cpu].list = t;
160 __cpu_raise_softirq(cpu, TASKLET_SOFTIRQ);
161 local_irq_enable();
162 }
163 }
164
165
166
167 struct tasklet_head tasklet_hi_vec[NR_CPUS] __cacheline_aligned;
168
169 static void tasklet_hi_action(struct softirq_action *a)
170 {
171 int cpu = smp_processor_id();
172 struct tasklet_struct *list;
173
174 local_irq_disable();
175 list = tasklet_hi_vec[cpu].list;
176 tasklet_hi_vec[cpu].list = NULL;
177 local_irq_enable();
178
179 while (list != NULL) {
180 struct tasklet_struct *t = list;
181
182 list = list->next;
183
184 if (tasklet_trylock(t)) {
185 if (atomic_read(&t->count) == 0) {
186 clear_bit(TASKLET_STATE_SCHED, &t->state);
187
188 t->func(t->data);
189 tasklet_unlock(t);
190 continue;
191 }
192 tasklet_unlock(t);
193 }
194 local_irq_disable();
195 t->next = tasklet_hi_vec[cpu].list;
196 tasklet_hi_vec[cpu].list = t;
197 __cpu_raise_softirq(cpu, HI_SOFTIRQ);
198 local_irq_enable();
199 }
200 }
201
202
203 void tasklet_init(struct tasklet_struct *t,
204 void (*func)(unsigned long), unsigned long data)
205 {
206 t->func = func;
207 t->data = data;
208 t->state = 0;
209 atomic_set(&t->count, 0);
210 }
211
212 void tasklet_kill(struct tasklet_struct *t)
213 {
214 if (in_interrupt())
215 printk("Attempt to kill tasklet from interrupt\n");
216
217 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
218 current->state = TASK_RUNNING;
219 do {
220 current->policy |= SCHED_YIELD;
221 schedule();
222 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
223 }
224 tasklet_unlock_wait(t);
225 clear_bit(TASKLET_STATE_SCHED, &t->state);
226 }
227
228
229
230 /* Old style BHs */
231
232 static void (*bh_base[32])(void);
233 struct tasklet_struct bh_task_vec[32];
234
235 /* BHs are serialized by spinlock global_bh_lock.
236
237 It is still possible to make synchronize_bh() as
238 spin_unlock_wait(&global_bh_lock). This operation is not used
239 by kernel now, so that this lock is not made private only
240 due to wait_on_irq().
241
242 It can be removed only after auditing all the BHs.
243 */
244 spinlock_t global_bh_lock = SPIN_LOCK_UNLOCKED;
245
246 static void bh_action(unsigned long nr)
247 {
248 int cpu = smp_processor_id();
249
250 if (!spin_trylock(&global_bh_lock))
251 goto resched;
252
253 if (!hardirq_trylock(cpu))
254 goto resched_unlock;
255
256 if (bh_base[nr])
257 bh_base[nr]();
258
259 hardirq_endlock(cpu);
260 spin_unlock(&global_bh_lock);
261 return;
262
263 resched_unlock:
264 spin_unlock(&global_bh_lock);
265 resched:
266 mark_bh(nr);
267 }
268
269 void init_bh(int nr, void (*routine)(void))
270 {
271 bh_base[nr] = routine;
272 mb();
273 }
274
275 void remove_bh(int nr)
276 {
277 tasklet_kill(bh_task_vec+nr);
278 bh_base[nr] = NULL;
279 }
280
281 void __init softirq_init()
282 {
283 int i;
284
285 for (i=0; i<32; i++)
286 tasklet_init(bh_task_vec+i, bh_action, i);
287
288 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
289 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
290 }
291
292 void __run_task_queue(task_queue *list)
293 {
294 struct list_head head, *next;
295 unsigned long flags;
296
297 spin_lock_irqsave(&tqueue_lock, flags);
298 list_add(&head, list);
299 list_del_init(list);
300 spin_unlock_irqrestore(&tqueue_lock, flags);
301
302 next = head.next;
303 while (next != &head) {
304 void (*f) (void *);
305 struct tq_struct *p;
306 void *data;
307
308 p = list_entry(next, struct tq_struct, list);
309 next = next->next;
310 f = p->routine;
311 data = p->data;
312 wmb();
313 p->sync = 0;
314 if (f)
315 f(data);
316 }
317 }
318
This page was automatically generated by the
LXR engine.
Visit the LXR main site for more
information.