0001
0002
0003
0004
0005 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
0006
0007 #include <linux/kernel.h>
0008 #include <linux/module.h>
0009 #include <linux/delay.h>
0010 #include <linux/sched.h>
0011 #include <linux/sched/signal.h>
0012 #include <linux/sched/clock.h>
0013 #include <linux/cpu.h>
0014 #include <linux/nmi.h>
0015 #include <linux/mm.h>
0016 #include <linux/uaccess.h>
0017 #include <linux/file.h>
0018
0019 static unsigned int time_secs;
0020 module_param(time_secs, uint, 0600);
0021 MODULE_PARM_DESC(time_secs, "lockup time in seconds, default 0");
0022
0023 static unsigned int time_nsecs;
0024 module_param(time_nsecs, uint, 0600);
0025 MODULE_PARM_DESC(time_nsecs, "nanoseconds part of lockup time, default 0");
0026
0027 static unsigned int cooldown_secs;
0028 module_param(cooldown_secs, uint, 0600);
0029 MODULE_PARM_DESC(cooldown_secs, "cooldown time between iterations in seconds, default 0");
0030
0031 static unsigned int cooldown_nsecs;
0032 module_param(cooldown_nsecs, uint, 0600);
0033 MODULE_PARM_DESC(cooldown_nsecs, "nanoseconds part of cooldown, default 0");
0034
0035 static unsigned int iterations = 1;
0036 module_param(iterations, uint, 0600);
0037 MODULE_PARM_DESC(iterations, "lockup iterations, default 1");
0038
0039 static bool all_cpus;
0040 module_param(all_cpus, bool, 0400);
0041 MODULE_PARM_DESC(all_cpus, "trigger lockup at all cpus at once");
0042
0043 static int wait_state;
0044 static char *state = "R";
0045 module_param(state, charp, 0400);
0046 MODULE_PARM_DESC(state, "wait in 'R' running (default), 'D' uninterruptible, 'K' killable, 'S' interruptible state");
0047
0048 static bool use_hrtimer;
0049 module_param(use_hrtimer, bool, 0400);
0050 MODULE_PARM_DESC(use_hrtimer, "use high-resolution timer for sleeping");
0051
0052 static bool iowait;
0053 module_param(iowait, bool, 0400);
0054 MODULE_PARM_DESC(iowait, "account sleep time as iowait");
0055
0056 static bool lock_read;
0057 module_param(lock_read, bool, 0400);
0058 MODULE_PARM_DESC(lock_read, "lock read-write locks for read");
0059
0060 static bool lock_single;
0061 module_param(lock_single, bool, 0400);
0062 MODULE_PARM_DESC(lock_single, "acquire locks only at one cpu");
0063
0064 static bool reacquire_locks;
0065 module_param(reacquire_locks, bool, 0400);
0066 MODULE_PARM_DESC(reacquire_locks, "release and reacquire locks/irq/preempt between iterations");
0067
0068 static bool touch_softlockup;
0069 module_param(touch_softlockup, bool, 0600);
0070 MODULE_PARM_DESC(touch_softlockup, "touch soft-lockup watchdog between iterations");
0071
0072 static bool touch_hardlockup;
0073 module_param(touch_hardlockup, bool, 0600);
0074 MODULE_PARM_DESC(touch_hardlockup, "touch hard-lockup watchdog between iterations");
0075
0076 static bool call_cond_resched;
0077 module_param(call_cond_resched, bool, 0600);
0078 MODULE_PARM_DESC(call_cond_resched, "call cond_resched() between iterations");
0079
0080 static bool measure_lock_wait;
0081 module_param(measure_lock_wait, bool, 0400);
0082 MODULE_PARM_DESC(measure_lock_wait, "measure lock wait time");
0083
0084 static unsigned long lock_wait_threshold = ULONG_MAX;
0085 module_param(lock_wait_threshold, ulong, 0400);
0086 MODULE_PARM_DESC(lock_wait_threshold, "print lock wait time longer than this in nanoseconds, default off");
0087
0088 static bool test_disable_irq;
0089 module_param_named(disable_irq, test_disable_irq, bool, 0400);
0090 MODULE_PARM_DESC(disable_irq, "disable interrupts: generate hard-lockups");
0091
0092 static bool disable_softirq;
0093 module_param(disable_softirq, bool, 0400);
0094 MODULE_PARM_DESC(disable_softirq, "disable bottom-half irq handlers");
0095
0096 static bool disable_preempt;
0097 module_param(disable_preempt, bool, 0400);
0098 MODULE_PARM_DESC(disable_preempt, "disable preemption: generate soft-lockups");
0099
0100 static bool lock_rcu;
0101 module_param(lock_rcu, bool, 0400);
0102 MODULE_PARM_DESC(lock_rcu, "grab rcu_read_lock: generate rcu stalls");
0103
0104 static bool lock_mmap_sem;
0105 module_param(lock_mmap_sem, bool, 0400);
0106 MODULE_PARM_DESC(lock_mmap_sem, "lock mm->mmap_lock: block procfs interfaces");
0107
0108 static unsigned long lock_rwsem_ptr;
0109 module_param_unsafe(lock_rwsem_ptr, ulong, 0400);
0110 MODULE_PARM_DESC(lock_rwsem_ptr, "lock rw_semaphore at address");
0111
0112 static unsigned long lock_mutex_ptr;
0113 module_param_unsafe(lock_mutex_ptr, ulong, 0400);
0114 MODULE_PARM_DESC(lock_mutex_ptr, "lock mutex at address");
0115
0116 static unsigned long lock_spinlock_ptr;
0117 module_param_unsafe(lock_spinlock_ptr, ulong, 0400);
0118 MODULE_PARM_DESC(lock_spinlock_ptr, "lock spinlock at address");
0119
0120 static unsigned long lock_rwlock_ptr;
0121 module_param_unsafe(lock_rwlock_ptr, ulong, 0400);
0122 MODULE_PARM_DESC(lock_rwlock_ptr, "lock rwlock at address");
0123
0124 static unsigned int alloc_pages_nr;
0125 module_param_unsafe(alloc_pages_nr, uint, 0600);
0126 MODULE_PARM_DESC(alloc_pages_nr, "allocate and free pages under locks");
0127
0128 static unsigned int alloc_pages_order;
0129 module_param(alloc_pages_order, uint, 0400);
0130 MODULE_PARM_DESC(alloc_pages_order, "page order to allocate");
0131
0132 static gfp_t alloc_pages_gfp = GFP_KERNEL;
0133 module_param_unsafe(alloc_pages_gfp, uint, 0400);
0134 MODULE_PARM_DESC(alloc_pages_gfp, "allocate pages with this gfp_mask, default GFP_KERNEL");
0135
0136 static bool alloc_pages_atomic;
0137 module_param(alloc_pages_atomic, bool, 0400);
0138 MODULE_PARM_DESC(alloc_pages_atomic, "allocate pages with GFP_ATOMIC");
0139
0140 static bool reallocate_pages;
0141 module_param(reallocate_pages, bool, 0400);
0142 MODULE_PARM_DESC(reallocate_pages, "free and allocate pages between iterations");
0143
0144 struct file *test_file;
0145 static struct inode *test_inode;
0146 static char test_file_path[256];
0147 module_param_string(file_path, test_file_path, sizeof(test_file_path), 0400);
0148 MODULE_PARM_DESC(file_path, "file path to test");
0149
0150 static bool test_lock_inode;
0151 module_param_named(lock_inode, test_lock_inode, bool, 0400);
0152 MODULE_PARM_DESC(lock_inode, "lock file -> inode -> i_rwsem");
0153
0154 static bool test_lock_mapping;
0155 module_param_named(lock_mapping, test_lock_mapping, bool, 0400);
0156 MODULE_PARM_DESC(lock_mapping, "lock file -> mapping -> i_mmap_rwsem");
0157
0158 static bool test_lock_sb_umount;
0159 module_param_named(lock_sb_umount, test_lock_sb_umount, bool, 0400);
0160 MODULE_PARM_DESC(lock_sb_umount, "lock file -> sb -> s_umount");
0161
0162 static atomic_t alloc_pages_failed = ATOMIC_INIT(0);
0163
0164 static atomic64_t max_lock_wait = ATOMIC64_INIT(0);
0165
0166 static struct task_struct *main_task;
0167 static int master_cpu;
0168
0169 static void test_lock(bool master, bool verbose)
0170 {
0171 u64 wait_start;
0172
0173 if (measure_lock_wait)
0174 wait_start = local_clock();
0175
0176 if (lock_mutex_ptr && master) {
0177 if (verbose)
0178 pr_notice("lock mutex %ps\n", (void *)lock_mutex_ptr);
0179 mutex_lock((struct mutex *)lock_mutex_ptr);
0180 }
0181
0182 if (lock_rwsem_ptr && master) {
0183 if (verbose)
0184 pr_notice("lock rw_semaphore %ps\n",
0185 (void *)lock_rwsem_ptr);
0186 if (lock_read)
0187 down_read((struct rw_semaphore *)lock_rwsem_ptr);
0188 else
0189 down_write((struct rw_semaphore *)lock_rwsem_ptr);
0190 }
0191
0192 if (lock_mmap_sem && master) {
0193 if (verbose)
0194 pr_notice("lock mmap_lock pid=%d\n", main_task->pid);
0195 if (lock_read)
0196 mmap_read_lock(main_task->mm);
0197 else
0198 mmap_write_lock(main_task->mm);
0199 }
0200
0201 if (test_disable_irq)
0202 local_irq_disable();
0203
0204 if (disable_softirq)
0205 local_bh_disable();
0206
0207 if (disable_preempt)
0208 preempt_disable();
0209
0210 if (lock_rcu)
0211 rcu_read_lock();
0212
0213 if (lock_spinlock_ptr && master) {
0214 if (verbose)
0215 pr_notice("lock spinlock %ps\n",
0216 (void *)lock_spinlock_ptr);
0217 spin_lock((spinlock_t *)lock_spinlock_ptr);
0218 }
0219
0220 if (lock_rwlock_ptr && master) {
0221 if (verbose)
0222 pr_notice("lock rwlock %ps\n",
0223 (void *)lock_rwlock_ptr);
0224 if (lock_read)
0225 read_lock((rwlock_t *)lock_rwlock_ptr);
0226 else
0227 write_lock((rwlock_t *)lock_rwlock_ptr);
0228 }
0229
0230 if (measure_lock_wait) {
0231 s64 cur_wait = local_clock() - wait_start;
0232 s64 max_wait = atomic64_read(&max_lock_wait);
0233
0234 do {
0235 if (cur_wait < max_wait)
0236 break;
0237 max_wait = atomic64_cmpxchg(&max_lock_wait,
0238 max_wait, cur_wait);
0239 } while (max_wait != cur_wait);
0240
0241 if (cur_wait > lock_wait_threshold)
0242 pr_notice_ratelimited("lock wait %lld ns\n", cur_wait);
0243 }
0244 }
0245
0246 static void test_unlock(bool master, bool verbose)
0247 {
0248 if (lock_rwlock_ptr && master) {
0249 if (lock_read)
0250 read_unlock((rwlock_t *)lock_rwlock_ptr);
0251 else
0252 write_unlock((rwlock_t *)lock_rwlock_ptr);
0253 if (verbose)
0254 pr_notice("unlock rwlock %ps\n",
0255 (void *)lock_rwlock_ptr);
0256 }
0257
0258 if (lock_spinlock_ptr && master) {
0259 spin_unlock((spinlock_t *)lock_spinlock_ptr);
0260 if (verbose)
0261 pr_notice("unlock spinlock %ps\n",
0262 (void *)lock_spinlock_ptr);
0263 }
0264
0265 if (lock_rcu)
0266 rcu_read_unlock();
0267
0268 if (disable_preempt)
0269 preempt_enable();
0270
0271 if (disable_softirq)
0272 local_bh_enable();
0273
0274 if (test_disable_irq)
0275 local_irq_enable();
0276
0277 if (lock_mmap_sem && master) {
0278 if (lock_read)
0279 mmap_read_unlock(main_task->mm);
0280 else
0281 mmap_write_unlock(main_task->mm);
0282 if (verbose)
0283 pr_notice("unlock mmap_lock pid=%d\n", main_task->pid);
0284 }
0285
0286 if (lock_rwsem_ptr && master) {
0287 if (lock_read)
0288 up_read((struct rw_semaphore *)lock_rwsem_ptr);
0289 else
0290 up_write((struct rw_semaphore *)lock_rwsem_ptr);
0291 if (verbose)
0292 pr_notice("unlock rw_semaphore %ps\n",
0293 (void *)lock_rwsem_ptr);
0294 }
0295
0296 if (lock_mutex_ptr && master) {
0297 mutex_unlock((struct mutex *)lock_mutex_ptr);
0298 if (verbose)
0299 pr_notice("unlock mutex %ps\n",
0300 (void *)lock_mutex_ptr);
0301 }
0302 }
0303
0304 static void test_alloc_pages(struct list_head *pages)
0305 {
0306 struct page *page;
0307 unsigned int i;
0308
0309 for (i = 0; i < alloc_pages_nr; i++) {
0310 page = alloc_pages(alloc_pages_gfp, alloc_pages_order);
0311 if (!page) {
0312 atomic_inc(&alloc_pages_failed);
0313 break;
0314 }
0315 list_add(&page->lru, pages);
0316 }
0317 }
0318
0319 static void test_free_pages(struct list_head *pages)
0320 {
0321 struct page *page, *next;
0322
0323 list_for_each_entry_safe(page, next, pages, lru)
0324 __free_pages(page, alloc_pages_order);
0325 INIT_LIST_HEAD(pages);
0326 }
0327
0328 static void test_wait(unsigned int secs, unsigned int nsecs)
0329 {
0330 if (wait_state == TASK_RUNNING) {
0331 if (secs)
0332 mdelay(secs * MSEC_PER_SEC);
0333 if (nsecs)
0334 ndelay(nsecs);
0335 return;
0336 }
0337
0338 __set_current_state(wait_state);
0339 if (use_hrtimer) {
0340 ktime_t time;
0341
0342 time = ns_to_ktime((u64)secs * NSEC_PER_SEC + nsecs);
0343 schedule_hrtimeout(&time, HRTIMER_MODE_REL);
0344 } else {
0345 schedule_timeout(secs * HZ + nsecs_to_jiffies(nsecs));
0346 }
0347 }
0348
0349 static void test_lockup(bool master)
0350 {
0351 u64 lockup_start = local_clock();
0352 unsigned int iter = 0;
0353 LIST_HEAD(pages);
0354
0355 pr_notice("Start on CPU%d\n", raw_smp_processor_id());
0356
0357 test_lock(master, true);
0358
0359 test_alloc_pages(&pages);
0360
0361 while (iter++ < iterations && !signal_pending(main_task)) {
0362
0363 if (iowait)
0364 current->in_iowait = 1;
0365
0366 test_wait(time_secs, time_nsecs);
0367
0368 if (iowait)
0369 current->in_iowait = 0;
0370
0371 if (reallocate_pages)
0372 test_free_pages(&pages);
0373
0374 if (reacquire_locks)
0375 test_unlock(master, false);
0376
0377 if (touch_softlockup)
0378 touch_softlockup_watchdog();
0379
0380 if (touch_hardlockup)
0381 touch_nmi_watchdog();
0382
0383 if (call_cond_resched)
0384 cond_resched();
0385
0386 test_wait(cooldown_secs, cooldown_nsecs);
0387
0388 if (reacquire_locks)
0389 test_lock(master, false);
0390
0391 if (reallocate_pages)
0392 test_alloc_pages(&pages);
0393 }
0394
0395 pr_notice("Finish on CPU%d in %lld ns\n", raw_smp_processor_id(),
0396 local_clock() - lockup_start);
0397
0398 test_free_pages(&pages);
0399
0400 test_unlock(master, true);
0401 }
0402
0403 static DEFINE_PER_CPU(struct work_struct, test_works);
0404
0405 static void test_work_fn(struct work_struct *work)
0406 {
0407 test_lockup(!lock_single ||
0408 work == per_cpu_ptr(&test_works, master_cpu));
0409 }
0410
0411 static bool test_kernel_ptr(unsigned long addr, int size)
0412 {
0413 void *ptr = (void *)addr;
0414 char buf;
0415
0416 if (!addr)
0417 return false;
0418
0419
0420 if (!IS_ENABLED(CONFIG_ALTERNATE_USER_ADDRESS_SPACE) &&
0421 (access_ok((void __user *)ptr, 1) ||
0422 access_ok((void __user *)ptr + size - 1, 1))) {
0423 pr_err("user space ptr invalid in kernel: %#lx\n", addr);
0424 return true;
0425 }
0426
0427 if (get_kernel_nofault(buf, ptr) ||
0428 get_kernel_nofault(buf, ptr + size - 1)) {
0429 pr_err("invalid kernel ptr: %#lx\n", addr);
0430 return true;
0431 }
0432
0433 return false;
0434 }
0435
0436 static bool __maybe_unused test_magic(unsigned long addr, int offset,
0437 unsigned int expected)
0438 {
0439 void *ptr = (void *)addr + offset;
0440 unsigned int magic = 0;
0441
0442 if (!addr)
0443 return false;
0444
0445 if (get_kernel_nofault(magic, ptr) || magic != expected) {
0446 pr_err("invalid magic at %#lx + %#x = %#x, expected %#x\n",
0447 addr, offset, magic, expected);
0448 return true;
0449 }
0450
0451 return false;
0452 }
0453
0454 static int __init test_lockup_init(void)
0455 {
0456 u64 test_start = local_clock();
0457
0458 main_task = current;
0459
0460 switch (state[0]) {
0461 case 'S':
0462 wait_state = TASK_INTERRUPTIBLE;
0463 break;
0464 case 'D':
0465 wait_state = TASK_UNINTERRUPTIBLE;
0466 break;
0467 case 'K':
0468 wait_state = TASK_KILLABLE;
0469 break;
0470 case 'R':
0471 wait_state = TASK_RUNNING;
0472 break;
0473 default:
0474 pr_err("unknown state=%s\n", state);
0475 return -EINVAL;
0476 }
0477
0478 if (alloc_pages_atomic)
0479 alloc_pages_gfp = GFP_ATOMIC;
0480
0481 if (test_kernel_ptr(lock_spinlock_ptr, sizeof(spinlock_t)) ||
0482 test_kernel_ptr(lock_rwlock_ptr, sizeof(rwlock_t)) ||
0483 test_kernel_ptr(lock_mutex_ptr, sizeof(struct mutex)) ||
0484 test_kernel_ptr(lock_rwsem_ptr, sizeof(struct rw_semaphore)))
0485 return -EINVAL;
0486
0487 #ifdef CONFIG_DEBUG_SPINLOCK
0488 #ifdef CONFIG_PREEMPT_RT
0489 if (test_magic(lock_spinlock_ptr,
0490 offsetof(spinlock_t, lock.wait_lock.magic),
0491 SPINLOCK_MAGIC) ||
0492 test_magic(lock_rwlock_ptr,
0493 offsetof(rwlock_t, rwbase.rtmutex.wait_lock.magic),
0494 SPINLOCK_MAGIC) ||
0495 test_magic(lock_mutex_ptr,
0496 offsetof(struct mutex, rtmutex.wait_lock.magic),
0497 SPINLOCK_MAGIC) ||
0498 test_magic(lock_rwsem_ptr,
0499 offsetof(struct rw_semaphore, rwbase.rtmutex.wait_lock.magic),
0500 SPINLOCK_MAGIC))
0501 return -EINVAL;
0502 #else
0503 if (test_magic(lock_spinlock_ptr,
0504 offsetof(spinlock_t, rlock.magic),
0505 SPINLOCK_MAGIC) ||
0506 test_magic(lock_rwlock_ptr,
0507 offsetof(rwlock_t, magic),
0508 RWLOCK_MAGIC) ||
0509 test_magic(lock_mutex_ptr,
0510 offsetof(struct mutex, wait_lock.magic),
0511 SPINLOCK_MAGIC) ||
0512 test_magic(lock_rwsem_ptr,
0513 offsetof(struct rw_semaphore, wait_lock.magic),
0514 SPINLOCK_MAGIC))
0515 return -EINVAL;
0516 #endif
0517 #endif
0518
0519 if ((wait_state != TASK_RUNNING ||
0520 (call_cond_resched && !reacquire_locks) ||
0521 (alloc_pages_nr && gfpflags_allow_blocking(alloc_pages_gfp))) &&
0522 (test_disable_irq || disable_softirq || disable_preempt ||
0523 lock_rcu || lock_spinlock_ptr || lock_rwlock_ptr)) {
0524 pr_err("refuse to sleep in atomic context\n");
0525 return -EINVAL;
0526 }
0527
0528 if (lock_mmap_sem && !main_task->mm) {
0529 pr_err("no mm to lock mmap_lock\n");
0530 return -EINVAL;
0531 }
0532
0533 if (test_file_path[0]) {
0534 test_file = filp_open(test_file_path, O_RDONLY, 0);
0535 if (IS_ERR(test_file)) {
0536 pr_err("failed to open %s: %ld\n", test_file_path, PTR_ERR(test_file));
0537 return PTR_ERR(test_file);
0538 }
0539 test_inode = file_inode(test_file);
0540 } else if (test_lock_inode ||
0541 test_lock_mapping ||
0542 test_lock_sb_umount) {
0543 pr_err("no file to lock\n");
0544 return -EINVAL;
0545 }
0546
0547 if (test_lock_inode && test_inode)
0548 lock_rwsem_ptr = (unsigned long)&test_inode->i_rwsem;
0549
0550 if (test_lock_mapping && test_file && test_file->f_mapping)
0551 lock_rwsem_ptr = (unsigned long)&test_file->f_mapping->i_mmap_rwsem;
0552
0553 if (test_lock_sb_umount && test_inode)
0554 lock_rwsem_ptr = (unsigned long)&test_inode->i_sb->s_umount;
0555
0556 pr_notice("START pid=%d time=%u +%u ns cooldown=%u +%u ns iterations=%u state=%s %s%s%s%s%s%s%s%s%s%s%s\n",
0557 main_task->pid, time_secs, time_nsecs,
0558 cooldown_secs, cooldown_nsecs, iterations, state,
0559 all_cpus ? "all_cpus " : "",
0560 iowait ? "iowait " : "",
0561 test_disable_irq ? "disable_irq " : "",
0562 disable_softirq ? "disable_softirq " : "",
0563 disable_preempt ? "disable_preempt " : "",
0564 lock_rcu ? "lock_rcu " : "",
0565 lock_read ? "lock_read " : "",
0566 touch_softlockup ? "touch_softlockup " : "",
0567 touch_hardlockup ? "touch_hardlockup " : "",
0568 call_cond_resched ? "call_cond_resched " : "",
0569 reacquire_locks ? "reacquire_locks " : "");
0570
0571 if (alloc_pages_nr)
0572 pr_notice("ALLOCATE PAGES nr=%u order=%u gfp=%pGg %s\n",
0573 alloc_pages_nr, alloc_pages_order, &alloc_pages_gfp,
0574 reallocate_pages ? "reallocate_pages " : "");
0575
0576 if (all_cpus) {
0577 unsigned int cpu;
0578
0579 cpus_read_lock();
0580
0581 preempt_disable();
0582 master_cpu = smp_processor_id();
0583 for_each_online_cpu(cpu) {
0584 INIT_WORK(per_cpu_ptr(&test_works, cpu), test_work_fn);
0585 queue_work_on(cpu, system_highpri_wq,
0586 per_cpu_ptr(&test_works, cpu));
0587 }
0588 preempt_enable();
0589
0590 for_each_online_cpu(cpu)
0591 flush_work(per_cpu_ptr(&test_works, cpu));
0592
0593 cpus_read_unlock();
0594 } else {
0595 test_lockup(true);
0596 }
0597
0598 if (measure_lock_wait)
0599 pr_notice("Maximum lock wait: %lld ns\n",
0600 atomic64_read(&max_lock_wait));
0601
0602 if (alloc_pages_nr)
0603 pr_notice("Page allocation failed %u times\n",
0604 atomic_read(&alloc_pages_failed));
0605
0606 pr_notice("FINISH in %llu ns\n", local_clock() - test_start);
0607
0608 if (test_file)
0609 fput(test_file);
0610
0611 if (signal_pending(main_task))
0612 return -EINTR;
0613
0614 return -EAGAIN;
0615 }
0616 module_init(test_lockup_init);
0617
0618 MODULE_LICENSE("GPL");
0619 MODULE_AUTHOR("Konstantin Khlebnikov <khlebnikov@yandex-team.ru>");
0620 MODULE_DESCRIPTION("Test module to generate lockups");
