x86/mm/mmio-mod.c

0001 // SPDX-License-Identifier: GPL-2.0-or-later
0002 /*
0003  *
0004  * Copyright (C) IBM Corporation, 2005
0005  *               Jeff Muizelaar, 2006, 2007
0006  *               Pekka Paalanen, 2008 <pq@iki.fi>
0007  *
0008  * Derived from the read-mod example from relay-examples by Tom Zanussi.
0009  */
0010
0011 #define pr_fmt(fmt) "mmiotrace: " fmt
0012
0013 #include <linux/moduleparam.h>
0014 #include <linux/debugfs.h>
0015 #include <linux/slab.h>
0016 #include <linux/uaccess.h>
0017 #include <linux/io.h>
0018 #include <linux/mmiotrace.h>
0019 #include <linux/pgtable.h>
0020 #include <asm/e820/api.h> /* for ISA_START_ADDRESS */
0021 #include <linux/atomic.h>
0022 #include <linux/percpu.h>
0023 #include <linux/cpu.h>
0024
0025 #include "pf_in.h"
0026
0027 struct trap_reason {
0028     unsigned long addr;
0029     unsigned long ip;
0030     enum reason_type type;
0031     int active_traces;
0032 };
0033
0034 struct remap_trace {
0035     struct list_head list;
0036     struct kmmio_probe probe;
0037     resource_size_t phys;
0038     unsigned long id;
0039 };
0040
0041 /* Accessed per-cpu. */
0042 static DEFINE_PER_CPU(struct trap_reason, pf_reason);
0043 static DEFINE_PER_CPU(struct mmiotrace_rw, cpu_trace);
0044
0045 static DEFINE_MUTEX(mmiotrace_mutex);
0046 static DEFINE_SPINLOCK(trace_lock);
0047 static atomic_t mmiotrace_enabled;
0048 static LIST_HEAD(trace_list);       /* struct remap_trace */
0049
0050 /*
0051  * Locking in this file:
0052  * - mmiotrace_mutex enforces enable/disable_mmiotrace() critical sections.
0053  * - mmiotrace_enabled may be modified only when holding mmiotrace_mutex
0054  *   and trace_lock.
0055  * - Routines depending on is_enabled() must take trace_lock.
0056  * - trace_list users must hold trace_lock.
0057  * - is_enabled() guarantees that mmio_trace_{rw,mapping} are allowed.
0058  * - pre/post callbacks assume the effect of is_enabled() being true.
0059  */
0060
0061 /* module parameters */
0062 static unsigned long    filter_offset;
0063 static bool     nommiotrace;
0064 static bool     trace_pc;
0065
0066 module_param(filter_offset, ulong, 0);
0067 module_param(nommiotrace, bool, 0);
0068 module_param(trace_pc, bool, 0);
0069
0070 MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
0071 MODULE_PARM_DESC(nommiotrace, "Disable actual MMIO tracing.");
0072 MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
0073
0074 static bool is_enabled(void)
0075 {
0076     return atomic_read(&mmiotrace_enabled);
0077 }
0078
0079 static void print_pte(unsigned long address)
0080 {
0081     unsigned int level;
0082     pte_t *pte = lookup_address(address, &level);
0083
0084     if (!pte) {
0085         pr_err("Error in %s: no pte for page 0x%08lx\n",
0086                __func__, address);
0087         return;
0088     }
0089
0090     if (level == PG_LEVEL_2M) {
0091         pr_emerg("4MB pages are not currently supported: 0x%08lx\n",
0092              address);
0093         BUG();
0094     }
0095     pr_info("pte for 0x%lx: 0x%llx 0x%llx\n",
0096         address,
0097         (unsigned long long)pte_val(*pte),
0098         (unsigned long long)pte_val(*pte) & _PAGE_PRESENT);
0099 }
0100
0101 /*
0102  * For some reason the pre/post pairs have been called in an
0103  * unmatched order. Report and die.
0104  */
0105 static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
0106 {
0107     const struct trap_reason *my_reason = &get_cpu_var(pf_reason);
0108     pr_emerg("unexpected fault for address: 0x%08lx, last fault for address: 0x%08lx\n",
0109          addr, my_reason->addr);
0110     print_pte(addr);
0111     pr_emerg("faulting IP is at %pS\n", (void *)regs->ip);
0112     pr_emerg("last faulting IP was at %pS\n", (void *)my_reason->ip);
0113 #ifdef __i386__
0114     pr_emerg("eax: %08lx   ebx: %08lx   ecx: %08lx   edx: %08lx\n",
0115          regs->ax, regs->bx, regs->cx, regs->dx);
0116     pr_emerg("esi: %08lx   edi: %08lx   ebp: %08lx   esp: %08lx\n",
0117          regs->si, regs->di, regs->bp, regs->sp);
0118 #else
0119     pr_emerg("rax: %016lx   rcx: %016lx   rdx: %016lx\n",
0120          regs->ax, regs->cx, regs->dx);
0121     pr_emerg("rsi: %016lx   rdi: %016lx   rbp: %016lx   rsp: %016lx\n",
0122          regs->si, regs->di, regs->bp, regs->sp);
0123 #endif
0124     put_cpu_var(pf_reason);
0125     BUG();
0126 }
0127
0128 static void pre(struct kmmio_probe *p, struct pt_regs *regs,
0129                         unsigned long addr)
0130 {
0131     struct trap_reason *my_reason = &get_cpu_var(pf_reason);
0132     struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
0133     const unsigned long instptr = instruction_pointer(regs);
0134     const enum reason_type type = get_ins_type(instptr);
0135     struct remap_trace *trace = p->private;
0136
0137     /* it doesn't make sense to have more than one active trace per cpu */
0138     if (my_reason->active_traces)
0139         die_kmmio_nesting_error(regs, addr);
0140     else
0141         my_reason->active_traces++;
0142
0143     my_reason->type = type;
0144     my_reason->addr = addr;
0145     my_reason->ip = instptr;
0146
0147     my_trace->phys = addr - trace->probe.addr + trace->phys;
0148     my_trace->map_id = trace->id;
0149
0150     /*
0151      * Only record the program counter when requested.
0152      * It may taint clean-room reverse engineering.
0153      */
0154     if (trace_pc)
0155         my_trace->pc = instptr;
0156     else
0157         my_trace->pc = 0;
0158
0159     /*
0160      * XXX: the timestamp recorded will be *after* the tracing has been
0161      * done, not at the time we hit the instruction. SMP implications
0162      * on event ordering?
0163      */
0164
0165     switch (type) {
0166     case REG_READ:
0167         my_trace->opcode = MMIO_READ;
0168         my_trace->width = get_ins_mem_width(instptr);
0169         break;
0170     case REG_WRITE:
0171         my_trace->opcode = MMIO_WRITE;
0172         my_trace->width = get_ins_mem_width(instptr);
0173         my_trace->value = get_ins_reg_val(instptr, regs);
0174         break;
0175     case IMM_WRITE:
0176         my_trace->opcode = MMIO_WRITE;
0177         my_trace->width = get_ins_mem_width(instptr);
0178         my_trace->value = get_ins_imm_val(instptr);
0179         break;
0180     default:
0181         {
0182             unsigned char *ip = (unsigned char *)instptr;
0183             my_trace->opcode = MMIO_UNKNOWN_OP;
0184             my_trace->width = 0;
0185             my_trace->value = (*ip) << 16 | *(ip + 1) << 8 |
0186                                 *(ip + 2);
0187         }
0188     }
0189     put_cpu_var(cpu_trace);
0190     put_cpu_var(pf_reason);
0191 }
0192
0193 static void post(struct kmmio_probe *p, unsigned long condition,
0194                             struct pt_regs *regs)
0195 {
0196     struct trap_reason *my_reason = &get_cpu_var(pf_reason);
0197     struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
0198
0199     /* this should always return the active_trace count to 0 */
0200     my_reason->active_traces--;
0201     if (my_reason->active_traces) {
0202         pr_emerg("unexpected post handler");
0203         BUG();
0204     }
0205
0206     switch (my_reason->type) {
0207     case REG_READ:
0208         my_trace->value = get_ins_reg_val(my_reason->ip, regs);
0209         break;
0210     default:
0211         break;
0212     }
0213
0214     mmio_trace_rw(my_trace);
0215     put_cpu_var(cpu_trace);
0216     put_cpu_var(pf_reason);
0217 }
0218
0219 static void ioremap_trace_core(resource_size_t offset, unsigned long size,
0220                             void __iomem *addr)
0221 {
0222     static atomic_t next_id;
0223     struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
0224     /* These are page-unaligned. */
0225     struct mmiotrace_map map = {
0226         .phys = offset,
0227         .virt = (unsigned long)addr,
0228         .len = size,
0229         .opcode = MMIO_PROBE
0230     };
0231
0232     if (!trace) {
0233         pr_err("kmalloc failed in ioremap\n");
0234         return;
0235     }
0236
0237     *trace = (struct remap_trace) {
0238         .probe = {
0239             .addr = (unsigned long)addr,
0240             .len = size,
0241             .pre_handler = pre,
0242             .post_handler = post,
0243             .private = trace
0244         },
0245         .phys = offset,
0246         .id = atomic_inc_return(&next_id)
0247     };
0248     map.map_id = trace->id;
0249
0250     spin_lock_irq(&trace_lock);
0251     if (!is_enabled()) {
0252         kfree(trace);
0253         goto not_enabled;
0254     }
0255
0256     mmio_trace_mapping(&map);
0257     list_add_tail(&trace->list, &trace_list);
0258     if (!nommiotrace)
0259         register_kmmio_probe(&trace->probe);
0260
0261 not_enabled:
0262     spin_unlock_irq(&trace_lock);
0263 }
0264
0265 void mmiotrace_ioremap(resource_size_t offset, unsigned long size,
0266                         void __iomem *addr)
0267 {
0268     if (!is_enabled()) /* recheck and proper locking in *_core() */
0269         return;
0270
0271     pr_debug("ioremap_*(0x%llx, 0x%lx) = %p\n",
0272          (unsigned long long)offset, size, addr);
0273     if ((filter_offset) && (offset != filter_offset))
0274         return;
0275     ioremap_trace_core(offset, size, addr);
0276 }
0277
0278 static void iounmap_trace_core(volatile void __iomem *addr)
0279 {
0280     struct mmiotrace_map map = {
0281         .phys = 0,
0282         .virt = (unsigned long)addr,
0283         .len = 0,
0284         .opcode = MMIO_UNPROBE
0285     };
0286     struct remap_trace *trace;
0287     struct remap_trace *tmp;
0288     struct remap_trace *found_trace = NULL;
0289
0290     pr_debug("Unmapping %p.\n", addr);
0291
0292     spin_lock_irq(&trace_lock);
0293     if (!is_enabled())
0294         goto not_enabled;
0295
0296     list_for_each_entry_safe(trace, tmp, &trace_list, list) {
0297         if ((unsigned long)addr == trace->probe.addr) {
0298             if (!nommiotrace)
0299                 unregister_kmmio_probe(&trace->probe);
0300             list_del(&trace->list);
0301             found_trace = trace;
0302             break;
0303         }
0304     }
0305     map.map_id = (found_trace) ? found_trace->id : -1;
0306     mmio_trace_mapping(&map);
0307
0308 not_enabled:
0309     spin_unlock_irq(&trace_lock);
0310     if (found_trace) {
0311         synchronize_rcu(); /* unregister_kmmio_probe() requirement */
0312         kfree(found_trace);
0313     }
0314 }
0315
0316 void mmiotrace_iounmap(volatile void __iomem *addr)
0317 {
0318     might_sleep();
0319     if (is_enabled()) /* recheck and proper locking in *_core() */
0320         iounmap_trace_core(addr);
0321 }
0322
0323 int mmiotrace_printk(const char *fmt, ...)
0324 {
0325     int ret = 0;
0326     va_list args;
0327     unsigned long flags;
0328     va_start(args, fmt);
0329
0330     spin_lock_irqsave(&trace_lock, flags);
0331     if (is_enabled())
0332         ret = mmio_trace_printk(fmt, args);
0333     spin_unlock_irqrestore(&trace_lock, flags);
0334
0335     va_end(args);
0336     return ret;
0337 }
0338 EXPORT_SYMBOL(mmiotrace_printk);
0339
0340 static void clear_trace_list(void)
0341 {
0342     struct remap_trace *trace;
0343     struct remap_trace *tmp;
0344
0345     /*
0346      * No locking required, because the caller ensures we are in a
0347      * critical section via mutex, and is_enabled() is false,
0348      * i.e. nothing can traverse or modify this list.
0349      * Caller also ensures is_enabled() cannot change.
0350      */
0351     list_for_each_entry(trace, &trace_list, list) {
0352         pr_notice("purging non-iounmapped trace @0x%08lx, size 0x%lx.\n",
0353               trace->probe.addr, trace->probe.len);
0354         if (!nommiotrace)
0355             unregister_kmmio_probe(&trace->probe);
0356     }
0357     synchronize_rcu(); /* unregister_kmmio_probe() requirement */
0358
0359     list_for_each_entry_safe(trace, tmp, &trace_list, list) {
0360         list_del(&trace->list);
0361         kfree(trace);
0362     }
0363 }
0364
0365 #ifdef CONFIG_HOTPLUG_CPU
0366 static cpumask_var_t downed_cpus;
0367
0368 static void enter_uniprocessor(void)
0369 {
0370     int cpu;
0371     int err;
0372
0373     if (!cpumask_available(downed_cpus) &&
0374         !alloc_cpumask_var(&downed_cpus, GFP_KERNEL)) {
0375         pr_notice("Failed to allocate mask\n");
0376         goto out;
0377     }
0378
0379     cpus_read_lock();
0380     cpumask_copy(downed_cpus, cpu_online_mask);
0381     cpumask_clear_cpu(cpumask_first(cpu_online_mask), downed_cpus);
0382     if (num_online_cpus() > 1)
0383         pr_notice("Disabling non-boot CPUs...\n");
0384     cpus_read_unlock();
0385
0386     for_each_cpu(cpu, downed_cpus) {
0387         err = remove_cpu(cpu);
0388         if (!err)
0389             pr_info("CPU%d is down.\n", cpu);
0390         else
0391             pr_err("Error taking CPU%d down: %d\n", cpu, err);
0392     }
0393 out:
0394     if (num_online_cpus() > 1)
0395         pr_warn("multiple CPUs still online, may miss events.\n");
0396 }
0397
0398 static void leave_uniprocessor(void)
0399 {
0400     int cpu;
0401     int err;
0402
0403     if (!cpumask_available(downed_cpus) || cpumask_empty(downed_cpus))
0404         return;
0405     pr_notice("Re-enabling CPUs...\n");
0406     for_each_cpu(cpu, downed_cpus) {
0407         err = add_cpu(cpu);
0408         if (!err)
0409             pr_info("enabled CPU%d.\n", cpu);
0410         else
0411             pr_err("cannot re-enable CPU%d: %d\n", cpu, err);
0412     }
0413 }
0414
0415 #else /* !CONFIG_HOTPLUG_CPU */
0416 static void enter_uniprocessor(void)
0417 {
0418     if (num_online_cpus() > 1)
0419         pr_warn("multiple CPUs are online, may miss events. "
0420             "Suggest booting with maxcpus=1 kernel argument.\n");
0421 }
0422
0423 static void leave_uniprocessor(void)
0424 {
0425 }
0426 #endif
0427
0428 void enable_mmiotrace(void)
0429 {
0430     mutex_lock(&mmiotrace_mutex);
0431     if (is_enabled())
0432         goto out;
0433
0434     if (nommiotrace)
0435         pr_info("MMIO tracing disabled.\n");
0436     kmmio_init();
0437     enter_uniprocessor();
0438     spin_lock_irq(&trace_lock);
0439     atomic_inc(&mmiotrace_enabled);
0440     spin_unlock_irq(&trace_lock);
0441     pr_info("enabled.\n");
0442 out:
0443     mutex_unlock(&mmiotrace_mutex);
0444 }
0445
0446 void disable_mmiotrace(void)
0447 {
0448     mutex_lock(&mmiotrace_mutex);
0449     if (!is_enabled())
0450         goto out;
0451
0452     spin_lock_irq(&trace_lock);
0453     atomic_dec(&mmiotrace_enabled);
0454     BUG_ON(is_enabled());
0455     spin_unlock_irq(&trace_lock);
0456
0457     clear_trace_list(); /* guarantees: no more kmmio callbacks */
0458     leave_uniprocessor();
0459     kmmio_cleanup();
0460     pr_info("disabled.\n");
0461 out:
0462     mutex_unlock(&mmiotrace_mutex);
0463 }