git.maemo.org Git - h-e-n/blob - kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/kernel_stat.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/init.h>
  15 #include <linux/mm.h>
  16 #include <linux/notifier.h>
  17 #include <linux/percpu.h>
  18 #include <linux/cpu.h>
  19 #include <linux/freezer.h>
  20 #include <linux/kthread.h>
  21 #include <linux/rcupdate.h>
  22 #include <linux/smp.h>
  23 #include <linux/tick.h>
  24
  25 #include <asm/irq.h>
  26 /*
  27    - No shared variables, all the data are CPU local.
  28    - If a softirq needs serialization, let it serialize itself
  29      by its own spinlocks.
  30    - Even if softirq is serialized, only local cpu is marked for
  31      execution. Hence, we get something sort of weak cpu binding.
  32      Though it is still not clear, will it result in better locality
  33      or will not.
  34
  35    Examples:
  36    - NET RX softirq. It is multithreaded and does not require
  37      any global serialization.
  38    - NET TX softirq. It kicks software netdevice queues, hence
  39      it is logically serialized per device, but this serialization
  40      is invisible to common code.
  41    - Tasklets: serialized wrt itself.
  42  */
  43
  44 #ifndef __ARCH_IRQ_STAT
  45 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  46 EXPORT_SYMBOL(irq_stat);
  47 #endif
  48
  49 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  50
  51 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  52
  53 /*
  54  * we cannot loop indefinitely here to avoid userspace starvation,
  55  * but we also don't want to introduce a worst case 1/HZ latency
  56  * to the pending events, so lets the scheduler to balance
  57  * the softirq load for us.
  58  */
  59 static inline void wakeup_softirqd(void)
  60 {
  61         /* Interrupts are disabled: no need to stop preemption */
  62         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  63
  64         if (tsk && tsk->state != TASK_RUNNING)
  65                 wake_up_process(tsk);
  66 }
  67
  68 /*
  69  * This one is for softirq.c-internal use,
  70  * where hardirqs are disabled legitimately:
  71  */
  72 #ifdef CONFIG_TRACE_IRQFLAGS
  73 static void __local_bh_disable(unsigned long ip)
  74 {
  75         unsigned long flags;
  76
  77         WARN_ON_ONCE(in_irq());
  78
  79         raw_local_irq_save(flags);
  80         add_preempt_count(SOFTIRQ_OFFSET);
  81         /*
  82          * Were softirqs turned off above:
  83          */
  84         if (softirq_count() == SOFTIRQ_OFFSET)
  85                 trace_softirqs_off(ip);
  86         raw_local_irq_restore(flags);
  87 }
  88 #else /* !CONFIG_TRACE_IRQFLAGS */
  89 static inline void __local_bh_disable(unsigned long ip)
  90 {
  91         add_preempt_count(SOFTIRQ_OFFSET);
  92         barrier();
  93 }
  94 #endif /* CONFIG_TRACE_IRQFLAGS */
  95
  96 void local_bh_disable(void)
  97 {
  98         __local_bh_disable((unsigned long)__builtin_return_address(0));
  99 }
 100
 101 EXPORT_SYMBOL(local_bh_disable);
 102
 103 void __local_bh_enable(void)
 104 {
 105         WARN_ON_ONCE(in_irq());
 106
 107         /*
 108          * softirqs should never be enabled by __local_bh_enable(),
 109          * it always nests inside local_bh_enable() sections:
 110          */
 111         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
 112
 113         sub_preempt_count(SOFTIRQ_OFFSET);
 114 }
 115 EXPORT_SYMBOL_GPL(__local_bh_enable);
 116
 117 /*
 118  * Special-case - softirqs can safely be enabled in
 119  * cond_resched_softirq(), or by __do_softirq(),
 120  * without processing still-pending softirqs:
 121  */
 122 void _local_bh_enable(void)
 123 {
 124         WARN_ON_ONCE(in_irq());
 125         WARN_ON_ONCE(!irqs_disabled());
 126
 127         if (softirq_count() == SOFTIRQ_OFFSET)
 128                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 129         sub_preempt_count(SOFTIRQ_OFFSET);
 130 }
 131
 132 EXPORT_SYMBOL(_local_bh_enable);
 133
 134 void local_bh_enable(void)
 135 {
 136 #ifdef CONFIG_TRACE_IRQFLAGS
 137         unsigned long flags;
 138
 139         WARN_ON_ONCE(in_irq());
 140 #endif
 141         WARN_ON_ONCE(irqs_disabled());
 142
 143 #ifdef CONFIG_TRACE_IRQFLAGS
 144         local_irq_save(flags);
 145 #endif
 146         /*
 147          * Are softirqs going to be turned on now:
 148          */
 149         if (softirq_count() == SOFTIRQ_OFFSET)
 150                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 151         /*
 152          * Keep preemption disabled until we are done with
 153          * softirq processing:
 154          */
 155         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 156
 157         if (unlikely(!in_interrupt() && local_softirq_pending()))
 158                 do_softirq();
 159
 160         dec_preempt_count();
 161 #ifdef CONFIG_TRACE_IRQFLAGS
 162         local_irq_restore(flags);
 163 #endif
 164         preempt_check_resched();
 165 }
 166 EXPORT_SYMBOL(local_bh_enable);
 167
 168 void local_bh_enable_ip(unsigned long ip)
 169 {
 170 #ifdef CONFIG_TRACE_IRQFLAGS
 171         unsigned long flags;
 172
 173         WARN_ON_ONCE(in_irq());
 174
 175         local_irq_save(flags);
 176 #endif
 177         /*
 178          * Are softirqs going to be turned on now:
 179          */
 180         if (softirq_count() == SOFTIRQ_OFFSET)
 181                 trace_softirqs_on(ip);
 182         /*
 183          * Keep preemption disabled until we are done with
 184          * softirq processing:
 185          */
 186         sub_preempt_count(SOFTIRQ_OFFSET - 1);
 187
 188         if (unlikely(!in_interrupt() && local_softirq_pending()))
 189                 do_softirq();
 190
 191         dec_preempt_count();
 192 #ifdef CONFIG_TRACE_IRQFLAGS
 193         local_irq_restore(flags);
 194 #endif
 195         preempt_check_resched();
 196 }
 197 EXPORT_SYMBOL(local_bh_enable_ip);
 198
 199 /*
 200  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 201  * and we fall back to softirqd after that.
 202  *
 203  * This number has been established via experimentation.
 204  * The two things to balance is latency against fairness -
 205  * we want to handle softirqs as soon as possible, but they
 206  * should not be able to lock up the box.
 207  */
 208 #define MAX_SOFTIRQ_RESTART 10
 209
 210 asmlinkage void __do_softirq(void)
 211 {
 212         struct softirq_action *h;
 213         __u32 pending;
 214         int max_restart = MAX_SOFTIRQ_RESTART;
 215         int cpu;
 216
 217         pending = local_softirq_pending();
 218         account_system_vtime(current);
 219
 220         __local_bh_disable((unsigned long)__builtin_return_address(0));
 221         trace_softirq_enter();
 222
 223         cpu = smp_processor_id();
 224 restart:
 225         /* Reset the pending bitmask before enabling irqs */
 226         set_softirq_pending(0);
 227
 228         local_irq_enable();
 229
 230         h = softirq_vec;
 231
 232         do {
 233                 if (pending & 1) {
 234                         h->action(h);
 235                         rcu_bh_qsctr_inc(cpu);
 236                 }
 237                 h++;
 238                 pending >>= 1;
 239         } while (pending);
 240
 241         local_irq_disable();
 242
 243         pending = local_softirq_pending();
 244         if (pending && --max_restart)
 245                 goto restart;
 246
 247         if (pending)
 248                 wakeup_softirqd();
 249
 250         trace_softirq_exit();
 251
 252         account_system_vtime(current);
 253         _local_bh_enable();
 254 }
 255
 256 #ifndef __ARCH_HAS_DO_SOFTIRQ
 257
 258 asmlinkage void do_softirq(void)
 259 {
 260         __u32 pending;
 261         unsigned long flags;
 262
 263         if (in_interrupt())
 264                 return;
 265
 266         local_irq_save(flags);
 267
 268         pending = local_softirq_pending();
 269
 270         if (pending)
 271                 __do_softirq();
 272
 273         local_irq_restore(flags);
 274 }
 275
 276 #endif
 277
 278 /*
 279  * Enter an interrupt context.
 280  */
 281 void irq_enter(void)
 282 {
 283 #ifdef CONFIG_NO_HZ
 284         int cpu = smp_processor_id();
 285         if (idle_cpu(cpu) && !in_interrupt())
 286                 tick_nohz_stop_idle(cpu);
 287 #endif
 288         __irq_enter();
 289 #ifdef CONFIG_NO_HZ
 290         if (idle_cpu(cpu))
 291                 tick_nohz_update_jiffies();
 292 #endif
 293 }
 294
 295 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 296 # define invoke_softirq()       __do_softirq()
 297 #else
 298 # define invoke_softirq()       do_softirq()
 299 #endif
 300
 301 /*
 302  * Exit an interrupt context. Process softirqs if needed and possible:
 303  */
 304 void irq_exit(void)
 305 {
 306         account_system_vtime(current);
 307         trace_hardirq_exit();
 308         sub_preempt_count(IRQ_EXIT_OFFSET);
 309         if (!in_interrupt() && local_softirq_pending())
 310                 invoke_softirq();
 311
 312 #ifdef CONFIG_NO_HZ
 313         /* Make sure that timer wheel updates are propagated */
 314         if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
 315                 tick_nohz_stop_sched_tick();
 316         rcu_irq_exit();
 317 #endif
 318         preempt_enable_no_resched();
 319 }
 320
 321 /*
 322  * This function must run with irqs disabled!
 323  */
 324 inline void raise_softirq_irqoff(unsigned int nr)
 325 {
 326         __raise_softirq_irqoff(nr);
 327
 328         /*
 329          * If we're in an interrupt or softirq, we're done
 330          * (this also catches softirq-disabled code). We will
 331          * actually run the softirq once we return from
 332          * the irq or softirq.
 333          *
 334          * Otherwise we wake up ksoftirqd to make sure we
 335          * schedule the softirq soon.
 336          */
 337         if (!in_interrupt())
 338                 wakeup_softirqd();
 339 }
 340
 341 void raise_softirq(unsigned int nr)
 342 {
 343         unsigned long flags;
 344
 345         local_irq_save(flags);
 346         raise_softirq_irqoff(nr);
 347         local_irq_restore(flags);
 348 }
 349
 350 void open_softirq(int nr, void (*action)(struct softirq_action *))
 351 {
 352         softirq_vec[nr].action = action;
 353 }
 354
 355 /* Tasklets */
 356 struct tasklet_head
 357 {
 358         struct tasklet_struct *head;
 359         struct tasklet_struct **tail;
 360 };
 361
 362 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 363 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 364
 365 void __tasklet_schedule(struct tasklet_struct *t)
 366 {
 367         unsigned long flags;
 368
 369         local_irq_save(flags);
 370         t->next = NULL;
 371         *__get_cpu_var(tasklet_vec).tail = t;
 372         __get_cpu_var(tasklet_vec).tail = &(t->next);
 373         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 374         local_irq_restore(flags);
 375 }
 376
 377 EXPORT_SYMBOL(__tasklet_schedule);
 378
 379 void __tasklet_hi_schedule(struct tasklet_struct *t)
 380 {
 381         unsigned long flags;
 382
 383         local_irq_save(flags);
 384         t->next = NULL;
 385         *__get_cpu_var(tasklet_hi_vec).tail = t;
 386         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 387         raise_softirq_irqoff(HI_SOFTIRQ);
 388         local_irq_restore(flags);
 389 }
 390
 391 EXPORT_SYMBOL(__tasklet_hi_schedule);
 392
 393 static void tasklet_action(struct softirq_action *a)
 394 {
 395         struct tasklet_struct *list;
 396
 397         local_irq_disable();
 398         list = __get_cpu_var(tasklet_vec).head;
 399         __get_cpu_var(tasklet_vec).head = NULL;
 400         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
 401         local_irq_enable();
 402
 403         while (list) {
 404                 struct tasklet_struct *t = list;
 405
 406                 list = list->next;
 407
 408                 if (tasklet_trylock(t)) {
 409                         if (!atomic_read(&t->count)) {
 410                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 411                                         BUG();
 412                                 t->func(t->data);
 413                                 tasklet_unlock(t);
 414                                 continue;
 415                         }
 416                         tasklet_unlock(t);
 417                 }
 418
 419                 local_irq_disable();
 420                 t->next = NULL;
 421                 *__get_cpu_var(tasklet_vec).tail = t;
 422                 __get_cpu_var(tasklet_vec).tail = &(t->next);
 423                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 424                 local_irq_enable();
 425         }
 426 }
 427
 428 static void tasklet_hi_action(struct softirq_action *a)
 429 {
 430         struct tasklet_struct *list;
 431
 432         local_irq_disable();
 433         list = __get_cpu_var(tasklet_hi_vec).head;
 434         __get_cpu_var(tasklet_hi_vec).head = NULL;
 435         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
 436         local_irq_enable();
 437
 438         while (list) {
 439                 struct tasklet_struct *t = list;
 440
 441                 list = list->next;
 442
 443                 if (tasklet_trylock(t)) {
 444                         if (!atomic_read(&t->count)) {
 445                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 446                                         BUG();
 447                                 t->func(t->data);
 448                                 tasklet_unlock(t);
 449                                 continue;
 450                         }
 451                         tasklet_unlock(t);
 452                 }
 453
 454                 local_irq_disable();
 455                 t->next = NULL;
 456                 *__get_cpu_var(tasklet_hi_vec).tail = t;
 457                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 458                 __raise_softirq_irqoff(HI_SOFTIRQ);
 459                 local_irq_enable();
 460         }
 461 }
 462
 463
 464 void tasklet_init(struct tasklet_struct *t,
 465                   void (*func)(unsigned long), unsigned long data)
 466 {
 467         t->next = NULL;
 468         t->state = 0;
 469         atomic_set(&t->count, 0);
 470         t->func = func;
 471         t->data = data;
 472 }
 473
 474 EXPORT_SYMBOL(tasklet_init);
 475
 476 void tasklet_kill(struct tasklet_struct *t)
 477 {
 478         if (in_interrupt())
 479                 printk("Attempt to kill tasklet from interrupt\n");
 480
 481         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 482                 do
 483                         yield();
 484                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 485         }
 486         tasklet_unlock_wait(t);
 487         clear_bit(TASKLET_STATE_SCHED, &t->state);
 488 }
 489
 490 EXPORT_SYMBOL(tasklet_kill);
 491
 492 void __init softirq_init(void)
 493 {
 494         int cpu;
 495
 496         for_each_possible_cpu(cpu) {
 497                 per_cpu(tasklet_vec, cpu).tail =
 498                         &per_cpu(tasklet_vec, cpu).head;
 499                 per_cpu(tasklet_hi_vec, cpu).tail =
 500                         &per_cpu(tasklet_hi_vec, cpu).head;
 501         }
 502
 503         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 504         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 505 }
 506
 507 static int ksoftirqd(void * __bind_cpu)
 508 {
 509         set_current_state(TASK_INTERRUPTIBLE);
 510
 511         while (!kthread_should_stop()) {
 512                 preempt_disable();
 513                 if (!local_softirq_pending()) {
 514                         preempt_enable_no_resched();
 515                         schedule();
 516                         preempt_disable();
 517                 }
 518
 519                 __set_current_state(TASK_RUNNING);
 520
 521                 while (local_softirq_pending()) {
 522                         /* Preempt disable stops cpu going offline.
 523                            If already offline, we'll be on wrong CPU:
 524                            don't process */
 525                         if (cpu_is_offline((long)__bind_cpu))
 526                                 goto wait_to_die;
 527                         do_softirq();
 528                         preempt_enable_no_resched();
 529                         cond_resched();
 530                         preempt_disable();
 531                 }
 532                 preempt_enable();
 533                 set_current_state(TASK_INTERRUPTIBLE);
 534         }
 535         __set_current_state(TASK_RUNNING);
 536         return 0;
 537
 538 wait_to_die:
 539         preempt_enable();
 540         /* Wait for kthread_stop */
 541         set_current_state(TASK_INTERRUPTIBLE);
 542         while (!kthread_should_stop()) {
 543                 schedule();
 544                 set_current_state(TASK_INTERRUPTIBLE);
 545         }
 546         __set_current_state(TASK_RUNNING);
 547         return 0;
 548 }
 549
 550 #ifdef CONFIG_HOTPLUG_CPU
 551 /*
 552  * tasklet_kill_immediate is called to remove a tasklet which can already be
 553  * scheduled for execution on @cpu.
 554  *
 555  * Unlike tasklet_kill, this function removes the tasklet
 556  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 557  *
 558  * When this function is called, @cpu must be in the CPU_DEAD state.
 559  */
 560 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 561 {
 562         struct tasklet_struct **i;
 563
 564         BUG_ON(cpu_online(cpu));
 565         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 566
 567         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 568                 return;
 569
 570         /* CPU is dead, so no lock needed. */
 571         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 572                 if (*i == t) {
 573                         *i = t->next;
 574                         /* If this was the tail element, move the tail ptr */
 575                         if (*i == NULL)
 576                                 per_cpu(tasklet_vec, cpu).tail = i;
 577                         return;
 578                 }
 579         }
 580         BUG();
 581 }
 582
 583 static void takeover_tasklets(unsigned int cpu)
 584 {
 585         /* CPU is dead, so no lock needed. */
 586         local_irq_disable();
 587
 588         /* Find end, append list for that CPU. */
 589         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 590                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
 591                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
 592                 per_cpu(tasklet_vec, cpu).head = NULL;
 593                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 594         }
 595         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 596
 597         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 598                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
 599                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
 600                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 601                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 602         }
 603         raise_softirq_irqoff(HI_SOFTIRQ);
 604
 605         local_irq_enable();
 606 }
 607 #endif /* CONFIG_HOTPLUG_CPU */
 608
 609 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 610                                   unsigned long action,
 611                                   void *hcpu)
 612 {
 613         int hotcpu = (unsigned long)hcpu;
 614         struct task_struct *p;
 615
 616         switch (action) {
 617         case CPU_UP_PREPARE:
 618         case CPU_UP_PREPARE_FROZEN:
 619                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 620                 if (IS_ERR(p)) {
 621                         printk("ksoftirqd for %i failed\n", hotcpu);
 622                         return NOTIFY_BAD;
 623                 }
 624                 kthread_bind(p, hotcpu);
 625                 per_cpu(ksoftirqd, hotcpu) = p;
 626                 break;
 627         case CPU_ONLINE:
 628         case CPU_ONLINE_FROZEN:
 629                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 630                 break;
 631 #ifdef CONFIG_HOTPLUG_CPU
 632         case CPU_UP_CANCELED:
 633         case CPU_UP_CANCELED_FROZEN:
 634                 if (!per_cpu(ksoftirqd, hotcpu))
 635                         break;
 636                 /* Unbind so it can run.  Fall thru. */
 637                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 638                              any_online_cpu(cpu_online_map));
 639         case CPU_DEAD:
 640         case CPU_DEAD_FROZEN: {
 641                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 642
 643                 p = per_cpu(ksoftirqd, hotcpu);
 644                 per_cpu(ksoftirqd, hotcpu) = NULL;
 645                 sched_setscheduler(p, SCHED_FIFO, &param);
 646                 kthread_stop(p);
 647                 takeover_tasklets(hotcpu);
 648                 break;
 649         }
 650 #endif /* CONFIG_HOTPLUG_CPU */
 651         }
 652         return NOTIFY_OK;
 653 }
 654
 655 static struct notifier_block __cpuinitdata cpu_nfb = {
 656         .notifier_call = cpu_callback
 657 };
 658
 659 __init int spawn_ksoftirqd(void)
 660 {
 661         void *cpu = (void *)(long)smp_processor_id();
 662         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 663
 664         BUG_ON(err == NOTIFY_BAD);
 665         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 666         register_cpu_notifier(&cpu_nfb);
 667         return 0;
 668 }
 669
 670 #ifdef CONFIG_SMP
 671 /*
 672  * Call a function on all processors
 673  */
 674 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
 675 {
 676         int ret = 0;
 677
 678         preempt_disable();
 679         ret = smp_call_function(func, info, retry, wait);
 680         local_irq_disable();
 681         func(info);
 682         local_irq_enable();
 683         preempt_enable();
 684         return ret;
 685 }
 686 EXPORT_SYMBOL(on_each_cpu);
 687 #endif