x86/kernel/doublefault_32.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #include <linux/mm.h>
0003 #include <linux/sched.h>
0004 #include <linux/sched/debug.h>
0005 #include <linux/init_task.h>
0006 #include <linux/fs.h>
0007
0008 #include <linux/uaccess.h>
0009 #include <asm/processor.h>
0010 #include <asm/desc.h>
0011 #include <asm/traps.h>
0012
0013 #define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM)
0014
0015 #define TSS(x) this_cpu_read(cpu_tss_rw.x86_tss.x)
0016
0017 static void set_df_gdt_entry(unsigned int cpu);
0018
0019 /*
0020  * Called by double_fault with CR0.TS and EFLAGS.NT cleared.  The CPU thinks
0021  * we're running the doublefault task.  Cannot return.
0022  */
0023 asmlinkage noinstr void __noreturn doublefault_shim(void)
0024 {
0025     unsigned long cr2;
0026     struct pt_regs regs;
0027
0028     BUILD_BUG_ON(sizeof(struct doublefault_stack) != PAGE_SIZE);
0029
0030     cr2 = native_read_cr2();
0031
0032     /* Reset back to the normal kernel task. */
0033     force_reload_TR();
0034     set_df_gdt_entry(smp_processor_id());
0035
0036     trace_hardirqs_off();
0037
0038     /*
0039      * Fill in pt_regs.  A downside of doing this in C is that the unwinder
0040      * won't see it (no ENCODE_FRAME_POINTER), so a nested stack dump
0041      * won't successfully unwind to the source of the double fault.
0042      * The main dump from exc_double_fault() is fine, though, since it
0043      * uses these regs directly.
0044      *
0045      * If anyone ever cares, this could be moved to asm.
0046      */
0047     regs.ss     = TSS(ss);
0048     regs.__ssh  = 0;
0049     regs.sp     = TSS(sp);
0050     regs.flags  = TSS(flags);
0051     regs.cs     = TSS(cs);
0052     /* We won't go through the entry asm, so we can leave __csh as 0. */
0053     regs.__csh  = 0;
0054     regs.ip     = TSS(ip);
0055     regs.orig_ax    = 0;
0056     regs.gs     = TSS(gs);
0057     regs.__gsh  = 0;
0058     regs.fs     = TSS(fs);
0059     regs.__fsh  = 0;
0060     regs.es     = TSS(es);
0061     regs.__esh  = 0;
0062     regs.ds     = TSS(ds);
0063     regs.__dsh  = 0;
0064     regs.ax     = TSS(ax);
0065     regs.bp     = TSS(bp);
0066     regs.di     = TSS(di);
0067     regs.si     = TSS(si);
0068     regs.dx     = TSS(dx);
0069     regs.cx     = TSS(cx);
0070     regs.bx     = TSS(bx);
0071
0072     exc_double_fault(&regs, 0, cr2);
0073
0074     /*
0075      * x86_32 does not save the original CR3 anywhere on a task switch.
0076      * This means that, even if we wanted to return, we would need to find
0077      * some way to reconstruct CR3.  We could make a credible guess based
0078      * on cpu_tlbstate, but that would be racy and would not account for
0079      * PTI.
0080      */
0081     panic("cannot return from double fault\n");
0082 }
0083
0084 DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = {
0085     .tss = {
0086                 /*
0087                  * No sp0 or ss0 -- we never run CPL != 0 with this TSS
0088                  * active.  sp is filled in later.
0089                  */
0090         .ldt        = 0,
0091     .io_bitmap_base = IO_BITMAP_OFFSET_INVALID,
0092
0093         .ip     = (unsigned long) asm_exc_double_fault,
0094         .flags      = X86_EFLAGS_FIXED,
0095         .es     = __USER_DS,
0096         .cs     = __KERNEL_CS,
0097         .ss     = __KERNEL_DS,
0098         .ds     = __USER_DS,
0099         .fs     = __KERNEL_PERCPU,
0100         .gs     = 0,
0101
0102         .__cr3      = __pa_nodebug(swapper_pg_dir),
0103     },
0104 };
0105
0106 static void set_df_gdt_entry(unsigned int cpu)
0107 {
0108     /* Set up doublefault TSS pointer in the GDT */
0109     __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS,
0110                &get_cpu_entry_area(cpu)->doublefault_stack.tss);
0111
0112 }
0113
0114 void doublefault_init_cpu_tss(void)
0115 {
0116     unsigned int cpu = smp_processor_id();
0117     struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
0118
0119     /*
0120      * The linker isn't smart enough to initialize percpu variables that
0121      * point to other places in percpu space.
0122      */
0123         this_cpu_write(doublefault_stack.tss.sp,
0124                        (unsigned long)&cea->doublefault_stack.stack +
0125                        sizeof(doublefault_stack.stack));
0126
0127     set_df_gdt_entry(cpu);
0128 }