x86/kernel/head_32.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  *
0004  *  Copyright (C) 1991, 1992  Linus Torvalds
0005  *
0006  *  Enhanced CPU detection and feature setting code by Mike Jagdis
0007  *  and Martin Mares, November 1997.
0008  */
0009
0010 .text
0011 #include <linux/threads.h>
0012 #include <linux/init.h>
0013 #include <linux/linkage.h>
0014 #include <asm/segment.h>
0015 #include <asm/page_types.h>
0016 #include <asm/pgtable_types.h>
0017 #include <asm/cache.h>
0018 #include <asm/thread_info.h>
0019 #include <asm/asm-offsets.h>
0020 #include <asm/setup.h>
0021 #include <asm/processor-flags.h>
0022 #include <asm/msr-index.h>
0023 #include <asm/cpufeatures.h>
0024 #include <asm/percpu.h>
0025 #include <asm/nops.h>
0026 #include <asm/nospec-branch.h>
0027 #include <asm/bootparam.h>
0028 #include <asm/export.h>
0029 #include <asm/pgtable_32.h>
0030
0031 /* Physical address */
0032 #define pa(X) ((X) - __PAGE_OFFSET)
0033
0034 /*
0035  * References to members of the new_cpu_data structure.
0036  */
0037
0038 #define X86     new_cpu_data+CPUINFO_x86
0039 #define X86_VENDOR  new_cpu_data+CPUINFO_x86_vendor
0040 #define X86_MODEL   new_cpu_data+CPUINFO_x86_model
0041 #define X86_STEPPING    new_cpu_data+CPUINFO_x86_stepping
0042 #define X86_HARD_MATH   new_cpu_data+CPUINFO_hard_math
0043 #define X86_CPUID   new_cpu_data+CPUINFO_cpuid_level
0044 #define X86_CAPABILITY  new_cpu_data+CPUINFO_x86_capability
0045 #define X86_VENDOR_ID   new_cpu_data+CPUINFO_x86_vendor_id
0046
0047
0048 #define SIZEOF_PTREGS 17*4
0049
0050 /*
0051  * Worst-case size of the kernel mapping we need to make:
0052  * a relocatable kernel can live anywhere in lowmem, so we need to be able
0053  * to map all of lowmem.
0054  */
0055 KERNEL_PAGES = LOWMEM_PAGES
0056
0057 INIT_MAP_SIZE = PAGE_TABLE_SIZE(KERNEL_PAGES) * PAGE_SIZE
0058 RESERVE_BRK(pagetables, INIT_MAP_SIZE)
0059
0060 /*
0061  * 32-bit kernel entrypoint; only used by the boot CPU.  On entry,
0062  * %esi points to the real-mode code as a 32-bit pointer.
0063  * CS and DS must be 4 GB flat segments, but we don't depend on
0064  * any particular GDT layout, because we load our own as soon as we
0065  * can.
0066  */
0067 __HEAD
0068 SYM_CODE_START(startup_32)
0069     movl pa(initial_stack),%ecx
0070
0071 /*
0072  * Set segments to known values.
0073  */
0074     lgdt pa(boot_gdt_descr)
0075     movl $(__BOOT_DS),%eax
0076     movl %eax,%ds
0077     movl %eax,%es
0078     movl %eax,%fs
0079     movl %eax,%gs
0080     movl %eax,%ss
0081     leal -__PAGE_OFFSET(%ecx),%esp
0082
0083 /*
0084  * Clear BSS first so that there are no surprises...
0085  */
0086     cld
0087     xorl %eax,%eax
0088     movl $pa(__bss_start),%edi
0089     movl $pa(__bss_stop),%ecx
0090     subl %edi,%ecx
0091     shrl $2,%ecx
0092     rep ; stosl
0093 /*
0094  * Copy bootup parameters out of the way.
0095  * Note: %esi still has the pointer to the real-mode data.
0096  * With the kexec as boot loader, parameter segment might be loaded beyond
0097  * kernel image and might not even be addressable by early boot page tables.
0098  * (kexec on panic case). Hence copy out the parameters before initializing
0099  * page tables.
0100  */
0101     movl $pa(boot_params),%edi
0102     movl $(PARAM_SIZE/4),%ecx
0103     cld
0104     rep
0105     movsl
0106     movl pa(boot_params) + NEW_CL_POINTER,%esi
0107     andl %esi,%esi
0108     jz 1f           # No command line
0109     movl $pa(boot_command_line),%edi
0110     movl $(COMMAND_LINE_SIZE/4),%ecx
0111     rep
0112     movsl
0113 1:
0114
0115 #ifdef CONFIG_OLPC
0116     /* save OFW's pgdir table for later use when calling into OFW */
0117     movl %cr3, %eax
0118     movl %eax, pa(olpc_ofw_pgd)
0119 #endif
0120
0121 #ifdef CONFIG_MICROCODE
0122     /* Early load ucode on BSP. */
0123     call load_ucode_bsp
0124 #endif
0125
0126     /* Create early pagetables. */
0127     call  mk_early_pgtbl_32
0128
0129     /* Do early initialization of the fixmap area */
0130     movl $pa(initial_pg_fixmap)+PDE_IDENT_ATTR,%eax
0131 #ifdef  CONFIG_X86_PAE
0132 #define KPMDS (((-__PAGE_OFFSET) >> 30) & 3) /* Number of kernel PMDs */
0133     movl %eax,pa(initial_pg_pmd+0x1000*KPMDS-8)
0134 #else
0135     movl %eax,pa(initial_page_table+0xffc)
0136 #endif
0137
0138     jmp .Ldefault_entry
0139 SYM_CODE_END(startup_32)
0140
0141 #ifdef CONFIG_HOTPLUG_CPU
0142 /*
0143  * Boot CPU0 entry point. It's called from play_dead(). Everything has been set
0144  * up already except stack. We just set up stack here. Then call
0145  * start_secondary().
0146  */
0147 SYM_FUNC_START(start_cpu0)
0148     movl initial_stack, %ecx
0149     movl %ecx, %esp
0150     call *(initial_code)
0151 1:  jmp 1b
0152 SYM_FUNC_END(start_cpu0)
0153 #endif
0154
0155 /*
0156  * Non-boot CPU entry point; entered from trampoline.S
0157  * We can't lgdt here, because lgdt itself uses a data segment, but
0158  * we know the trampoline has already loaded the boot_gdt for us.
0159  *
0160  * If cpu hotplug is not supported then this code can go in init section
0161  * which will be freed later
0162  */
0163 SYM_FUNC_START(startup_32_smp)
0164     cld
0165     movl $(__BOOT_DS),%eax
0166     movl %eax,%ds
0167     movl %eax,%es
0168     movl %eax,%fs
0169     movl %eax,%gs
0170     movl pa(initial_stack),%ecx
0171     movl %eax,%ss
0172     leal -__PAGE_OFFSET(%ecx),%esp
0173
0174 #ifdef CONFIG_MICROCODE
0175     /* Early load ucode on AP. */
0176     call load_ucode_ap
0177 #endif
0178
0179 .Ldefault_entry:
0180     movl $(CR0_STATE & ~X86_CR0_PG),%eax
0181     movl %eax,%cr0
0182
0183 /*
0184  * We want to start out with EFLAGS unambiguously cleared. Some BIOSes leave
0185  * bits like NT set. This would confuse the debugger if this code is traced. So
0186  * initialize them properly now before switching to protected mode. That means
0187  * DF in particular (even though we have cleared it earlier after copying the
0188  * command line) because GCC expects it.
0189  */
0190     pushl $0
0191     popfl
0192
0193 /*
0194  * New page tables may be in 4Mbyte page mode and may be using the global pages.
0195  *
0196  * NOTE! If we are on a 486 we may have no cr4 at all! Specifically, cr4 exists
0197  * if and only if CPUID exists and has flags other than the FPU flag set.
0198  */
0199     movl $-1,pa(X86_CPUID)      # preset CPUID level
0200     movl $X86_EFLAGS_ID,%ecx
0201     pushl %ecx
0202     popfl               # set EFLAGS=ID
0203     pushfl
0204     popl %eax           # get EFLAGS
0205     testl $X86_EFLAGS_ID,%eax   # did EFLAGS.ID remained set?
0206     jz .Lenable_paging      # hw disallowed setting of ID bit
0207                     # which means no CPUID and no CR4
0208
0209     xorl %eax,%eax
0210     cpuid
0211     movl %eax,pa(X86_CPUID)     # save largest std CPUID function
0212
0213     movl $1,%eax
0214     cpuid
0215     andl $~1,%edx           # Ignore CPUID.FPU
0216     jz .Lenable_paging      # No flags or only CPUID.FPU = no CR4
0217
0218     movl pa(mmu_cr4_features),%eax
0219     movl %eax,%cr4
0220
0221     testb $X86_CR4_PAE, %al     # check if PAE is enabled
0222     jz .Lenable_paging
0223
0224     /* Check if extended functions are implemented */
0225     movl $0x80000000, %eax
0226     cpuid
0227     /* Value must be in the range 0x80000001 to 0x8000ffff */
0228     subl $0x80000001, %eax
0229     cmpl $(0x8000ffff-0x80000001), %eax
0230     ja .Lenable_paging
0231
0232     /* Clear bogus XD_DISABLE bits */
0233     call verify_cpu
0234
0235     mov $0x80000001, %eax
0236     cpuid
0237     /* Execute Disable bit supported? */
0238     btl $(X86_FEATURE_NX & 31), %edx
0239     jnc .Lenable_paging
0240
0241     /* Setup EFER (Extended Feature Enable Register) */
0242     movl $MSR_EFER, %ecx
0243     rdmsr
0244
0245     btsl $_EFER_NX, %eax
0246     /* Make changes effective */
0247     wrmsr
0248
0249 .Lenable_paging:
0250
0251 /*
0252  * Enable paging
0253  */
0254     movl $pa(initial_page_table), %eax
0255     movl %eax,%cr3      /* set the page table pointer.. */
0256     movl $CR0_STATE,%eax
0257     movl %eax,%cr0      /* ..and set paging (PG) bit */
0258     ljmp $__BOOT_CS,$1f /* Clear prefetch and normalize %eip */
0259 1:
0260     /* Shift the stack pointer to a virtual address */
0261     addl $__PAGE_OFFSET, %esp
0262
0263 /*
0264  * start system 32-bit setup. We need to re-do some of the things done
0265  * in 16-bit mode for the "real" operations.
0266  */
0267     movl setup_once_ref,%eax
0268     andl %eax,%eax
0269     jz 1f               # Did we do this already?
0270     call *%eax
0271 1:
0272
0273 /*
0274  * Check if it is 486
0275  */
0276     movb $4,X86         # at least 486
0277     cmpl $-1,X86_CPUID
0278     je .Lis486
0279
0280     /* get vendor info */
0281     xorl %eax,%eax          # call CPUID with 0 -> return vendor ID
0282     cpuid
0283     movl %eax,X86_CPUID     # save CPUID level
0284     movl %ebx,X86_VENDOR_ID     # lo 4 chars
0285     movl %edx,X86_VENDOR_ID+4   # next 4 chars
0286     movl %ecx,X86_VENDOR_ID+8   # last 4 chars
0287
0288     orl %eax,%eax           # do we have processor info as well?
0289     je .Lis486
0290
0291     movl $1,%eax        # Use the CPUID instruction to get CPU type
0292     cpuid
0293     movb %al,%cl        # save reg for future use
0294     andb $0x0f,%ah      # mask processor family
0295     movb %ah,X86
0296     andb $0xf0,%al      # mask model
0297     shrb $4,%al
0298     movb %al,X86_MODEL
0299     andb $0x0f,%cl      # mask mask revision
0300     movb %cl,X86_STEPPING
0301     movl %edx,X86_CAPABILITY
0302
0303 .Lis486:
0304     movl $0x50022,%ecx  # set AM, WP, NE and MP
0305     movl %cr0,%eax
0306     andl $0x80000011,%eax   # Save PG,PE,ET
0307     orl %ecx,%eax
0308     movl %eax,%cr0
0309
0310     lgdt early_gdt_descr
0311     ljmp $(__KERNEL_CS),$1f
0312 1:  movl $(__KERNEL_DS),%eax    # reload all the segment registers
0313     movl %eax,%ss           # after changing gdt.
0314
0315     movl $(__USER_DS),%eax      # DS/ES contains default USER segment
0316     movl %eax,%ds
0317     movl %eax,%es
0318
0319     movl $(__KERNEL_PERCPU), %eax
0320     movl %eax,%fs           # set this cpu's percpu
0321
0322     xorl %eax,%eax
0323     movl %eax,%gs           # clear possible garbage in %gs
0324
0325     xorl %eax,%eax          # Clear LDT
0326     lldt %ax
0327
0328     call *(initial_code)
0329 1:  jmp 1b
0330 SYM_FUNC_END(startup_32_smp)
0331
0332 #include "verify_cpu.S"
0333
0334 /*
0335  *  setup_once
0336  *
0337  *  The setup work we only want to run on the BSP.
0338  *
0339  *  Warning: %esi is live across this function.
0340  */
0341 __INIT
0342 setup_once:
0343     andl $0,setup_once_ref  /* Once is enough, thanks */
0344     RET
0345
0346 SYM_FUNC_START(early_idt_handler_array)
0347     # 36(%esp) %eflags
0348     # 32(%esp) %cs
0349     # 28(%esp) %eip
0350     # 24(%rsp) error code
0351     i = 0
0352     .rept NUM_EXCEPTION_VECTORS
0353     .if ((EXCEPTION_ERRCODE_MASK >> i) & 1) == 0
0354     pushl $0        # Dummy error code, to make stack frame uniform
0355     .endif
0356     pushl $i        # 20(%esp) Vector number
0357     jmp early_idt_handler_common
0358     i = i + 1
0359     .fill early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
0360     .endr
0361 SYM_FUNC_END(early_idt_handler_array)
0362
0363 SYM_CODE_START_LOCAL(early_idt_handler_common)
0364     /*
0365      * The stack is the hardware frame, an error code or zero, and the
0366      * vector number.
0367      */
0368     cld
0369
0370     incl %ss:early_recursion_flag
0371
0372     /* The vector number is in pt_regs->gs */
0373
0374     cld
0375     pushl   %fs     /* pt_regs->fs (__fsh varies by model) */
0376     pushl   %es     /* pt_regs->es (__esh varies by model) */
0377     pushl   %ds     /* pt_regs->ds (__dsh varies by model) */
0378     pushl   %eax        /* pt_regs->ax */
0379     pushl   %ebp        /* pt_regs->bp */
0380     pushl   %edi        /* pt_regs->di */
0381     pushl   %esi        /* pt_regs->si */
0382     pushl   %edx        /* pt_regs->dx */
0383     pushl   %ecx        /* pt_regs->cx */
0384     pushl   %ebx        /* pt_regs->bx */
0385
0386     /* Fix up DS and ES */
0387     movl    $(__KERNEL_DS), %ecx
0388     movl    %ecx, %ds
0389     movl    %ecx, %es
0390
0391     /* Load the vector number into EDX */
0392     movl    PT_GS(%esp), %edx
0393
0394     /* Load GS into pt_regs->gs (and maybe clobber __gsh) */
0395     movw    %gs, PT_GS(%esp)
0396
0397     movl    %esp, %eax  /* args are pt_regs (EAX), trapnr (EDX) */
0398     call    early_fixup_exception
0399
0400     popl    %ebx        /* pt_regs->bx */
0401     popl    %ecx        /* pt_regs->cx */
0402     popl    %edx        /* pt_regs->dx */
0403     popl    %esi        /* pt_regs->si */
0404     popl    %edi        /* pt_regs->di */
0405     popl    %ebp        /* pt_regs->bp */
0406     popl    %eax        /* pt_regs->ax */
0407     popl    %ds     /* pt_regs->ds (always ignores __dsh) */
0408     popl    %es     /* pt_regs->es (always ignores __esh) */
0409     popl    %fs     /* pt_regs->fs (always ignores __fsh) */
0410     popl    %gs     /* pt_regs->gs (always ignores __gsh) */
0411     decl    %ss:early_recursion_flag
0412     addl    $4, %esp    /* pop pt_regs->orig_ax */
0413     iret
0414 SYM_CODE_END(early_idt_handler_common)
0415
0416 /* This is the default interrupt "handler" :-) */
0417 SYM_FUNC_START(early_ignore_irq)
0418     cld
0419 #ifdef CONFIG_PRINTK
0420     pushl %eax
0421     pushl %ecx
0422     pushl %edx
0423     pushl %es
0424     pushl %ds
0425     movl $(__KERNEL_DS),%eax
0426     movl %eax,%ds
0427     movl %eax,%es
0428     cmpl $2,early_recursion_flag
0429     je hlt_loop
0430     incl early_recursion_flag
0431     pushl 16(%esp)
0432     pushl 24(%esp)
0433     pushl 32(%esp)
0434     pushl 40(%esp)
0435     pushl $int_msg
0436     call _printk
0437
0438     call dump_stack
0439
0440     addl $(5*4),%esp
0441     popl %ds
0442     popl %es
0443     popl %edx
0444     popl %ecx
0445     popl %eax
0446 #endif
0447     iret
0448
0449 hlt_loop:
0450     hlt
0451     jmp hlt_loop
0452 SYM_FUNC_END(early_ignore_irq)
0453
0454 __INITDATA
0455     .align 4
0456 SYM_DATA(early_recursion_flag, .long 0)
0457
0458 __REFDATA
0459     .align 4
0460 SYM_DATA(initial_code,      .long i386_start_kernel)
0461 SYM_DATA(setup_once_ref,    .long setup_once)
0462
0463 #ifdef CONFIG_PAGE_TABLE_ISOLATION
0464 #define PGD_ALIGN   (2 * PAGE_SIZE)
0465 #define PTI_USER_PGD_FILL   1024
0466 #else
0467 #define PGD_ALIGN   (PAGE_SIZE)
0468 #define PTI_USER_PGD_FILL   0
0469 #endif
0470 /*
0471  * BSS section
0472  */
0473 __PAGE_ALIGNED_BSS
0474     .align PGD_ALIGN
0475 #ifdef CONFIG_X86_PAE
0476 .globl initial_pg_pmd
0477 initial_pg_pmd:
0478     .fill 1024*KPMDS,4,0
0479 #else
0480 .globl initial_page_table
0481 initial_page_table:
0482     .fill 1024,4,0
0483 #endif
0484     .align PGD_ALIGN
0485 initial_pg_fixmap:
0486     .fill 1024,4,0
0487 .globl swapper_pg_dir
0488     .align PGD_ALIGN
0489 swapper_pg_dir:
0490     .fill 1024,4,0
0491     .fill PTI_USER_PGD_FILL,4,0
0492 .globl empty_zero_page
0493 empty_zero_page:
0494     .fill 4096,1,0
0495 EXPORT_SYMBOL(empty_zero_page)
0496
0497 /*
0498  * This starts the data section.
0499  */
0500 #ifdef CONFIG_X86_PAE
0501 __PAGE_ALIGNED_DATA
0502     /* Page-aligned for the benefit of paravirt? */
0503     .align PGD_ALIGN
0504 SYM_DATA_START(initial_page_table)
0505     .long   pa(initial_pg_pmd+PGD_IDENT_ATTR),0 /* low identity map */
0506 # if KPMDS == 3
0507     .long   pa(initial_pg_pmd+PGD_IDENT_ATTR),0
0508     .long   pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0
0509     .long   pa(initial_pg_pmd+PGD_IDENT_ATTR+0x2000),0
0510 # elif KPMDS == 2
0511     .long   0,0
0512     .long   pa(initial_pg_pmd+PGD_IDENT_ATTR),0
0513     .long   pa(initial_pg_pmd+PGD_IDENT_ATTR+0x1000),0
0514 # elif KPMDS == 1
0515     .long   0,0
0516     .long   0,0
0517     .long   pa(initial_pg_pmd+PGD_IDENT_ATTR),0
0518 # else
0519 #  error "Kernel PMDs should be 1, 2 or 3"
0520 # endif
0521     .align PAGE_SIZE        /* needs to be page-sized too */
0522
0523 #ifdef CONFIG_PAGE_TABLE_ISOLATION
0524     /*
0525      * PTI needs another page so sync_initial_pagetable() works correctly
0526      * and does not scribble over the data which is placed behind the
0527      * actual initial_page_table. See clone_pgd_range().
0528      */
0529     .fill 1024, 4, 0
0530 #endif
0531
0532 SYM_DATA_END(initial_page_table)
0533 #endif
0534
0535 .data
0536 .balign 4
0537 /*
0538  * The SIZEOF_PTREGS gap is a convention which helps the in-kernel unwinder
0539  * reliably detect the end of the stack.
0540  */
0541 SYM_DATA(initial_stack,
0542         .long init_thread_union + THREAD_SIZE -
0543         SIZEOF_PTREGS - TOP_OF_KERNEL_STACK_PADDING)
0544
0545 __INITRODATA
0546 int_msg:
0547     .asciz "Unknown interrupt or fault at: %p %p %p\n"
0548
0549 #include "../../x86/xen/xen-head.S"
0550
0551 /*
0552  * The IDT and GDT 'descriptors' are a strange 48-bit object
0553  * only used by the lidt and lgdt instructions. They are not
0554  * like usual segment descriptors - they consist of a 16-bit
0555  * segment size, and 32-bit linear address value:
0556  */
0557
0558     .data
0559     ALIGN
0560 # early boot GDT descriptor (must use 1:1 address mapping)
0561     .word 0             # 32 bit align gdt_desc.address
0562 SYM_DATA_START_LOCAL(boot_gdt_descr)
0563     .word __BOOT_DS+7
0564     .long boot_gdt - __PAGE_OFFSET
0565 SYM_DATA_END(boot_gdt_descr)
0566
0567 # boot GDT descriptor (later on used by CPU#0):
0568     .word 0             # 32 bit align gdt_desc.address
0569 SYM_DATA_START(early_gdt_descr)
0570     .word GDT_ENTRIES*8-1
0571     .long gdt_page          /* Overwritten for secondary CPUs */
0572 SYM_DATA_END(early_gdt_descr)
0573
0574 /*
0575  * The boot_gdt must mirror the equivalent in setup.S and is
0576  * used only for booting.
0577  */
0578     .align L1_CACHE_BYTES
0579 SYM_DATA_START(boot_gdt)
0580     .fill GDT_ENTRY_BOOT_CS,8,0
0581     .quad 0x00cf9a000000ffff    /* kernel 4GB code at 0x00000000 */
0582     .quad 0x00cf92000000ffff    /* kernel 4GB data at 0x00000000 */
0583 SYM_DATA_END(boot_gdt)