x86/kernel/vmlinux.lds.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * ld script for the x86 kernel
0004  *
0005  * Historic 32-bit version written by Martin Mares <mj@atrey.karlin.mff.cuni.cz>
0006  *
0007  * Modernisation, unification and other changes and fixes:
0008  *   Copyright (C) 2007-2009  Sam Ravnborg <sam@ravnborg.org>
0009  *
0010  *
0011  * Don't define absolute symbols until and unless you know that symbol
0012  * value is should remain constant even if kernel image is relocated
0013  * at run time. Absolute symbols are not relocated. If symbol value should
0014  * change if kernel is relocated, make the symbol section relative and
0015  * put it inside the section definition.
0016  */
0017
0018 #ifdef CONFIG_X86_32
0019 #define LOAD_OFFSET __PAGE_OFFSET
0020 #else
0021 #define LOAD_OFFSET __START_KERNEL_map
0022 #endif
0023
0024 #define RUNTIME_DISCARD_EXIT
0025 #define EMITS_PT_NOTE
0026 #define RO_EXCEPTION_TABLE_ALIGN    16
0027
0028 #include <asm-generic/vmlinux.lds.h>
0029 #include <asm/asm-offsets.h>
0030 #include <asm/thread_info.h>
0031 #include <asm/page_types.h>
0032 #include <asm/orc_lookup.h>
0033 #include <asm/cache.h>
0034 #include <asm/boot.h>
0035
0036 #undef i386     /* in case the preprocessor is a 32bit one */
0037
0038 OUTPUT_FORMAT(CONFIG_OUTPUT_FORMAT)
0039
0040 #ifdef CONFIG_X86_32
0041 OUTPUT_ARCH(i386)
0042 ENTRY(phys_startup_32)
0043 #else
0044 OUTPUT_ARCH(i386:x86-64)
0045 ENTRY(phys_startup_64)
0046 #endif
0047
0048 jiffies = jiffies_64;
0049
0050 #if defined(CONFIG_X86_64)
0051 /*
0052  * On 64-bit, align RODATA to 2MB so we retain large page mappings for
0053  * boundaries spanning kernel text, rodata and data sections.
0054  *
0055  * However, kernel identity mappings will have different RWX permissions
0056  * to the pages mapping to text and to the pages padding (which are freed) the
0057  * text section. Hence kernel identity mappings will be broken to smaller
0058  * pages. For 64-bit, kernel text and kernel identity mappings are different,
0059  * so we can enable protection checks as well as retain 2MB large page
0060  * mappings for kernel text.
0061  */
0062 #define X86_ALIGN_RODATA_BEGIN  . = ALIGN(HPAGE_SIZE);
0063
0064 #define X86_ALIGN_RODATA_END                    \
0065         . = ALIGN(HPAGE_SIZE);              \
0066         __end_rodata_hpage_align = .;           \
0067         __end_rodata_aligned = .;
0068
0069 #define ALIGN_ENTRY_TEXT_BEGIN  . = ALIGN(PMD_SIZE);
0070 #define ALIGN_ENTRY_TEXT_END    . = ALIGN(PMD_SIZE);
0071
0072 /*
0073  * This section contains data which will be mapped as decrypted. Memory
0074  * encryption operates on a page basis. Make this section PMD-aligned
0075  * to avoid splitting the pages while mapping the section early.
0076  *
0077  * Note: We use a separate section so that only this section gets
0078  * decrypted to avoid exposing more than we wish.
0079  */
0080 #define BSS_DECRYPTED                       \
0081     . = ALIGN(PMD_SIZE);                    \
0082     __start_bss_decrypted = .;              \
0083     *(.bss..decrypted);                 \
0084     . = ALIGN(PAGE_SIZE);                   \
0085     __start_bss_decrypted_unused = .;           \
0086     . = ALIGN(PMD_SIZE);                    \
0087     __end_bss_decrypted = .;                \
0088
0089 #else
0090
0091 #define X86_ALIGN_RODATA_BEGIN
0092 #define X86_ALIGN_RODATA_END                    \
0093         . = ALIGN(PAGE_SIZE);               \
0094         __end_rodata_aligned = .;
0095
0096 #define ALIGN_ENTRY_TEXT_BEGIN
0097 #define ALIGN_ENTRY_TEXT_END
0098 #define BSS_DECRYPTED
0099
0100 #endif
0101
0102 PHDRS {
0103     text PT_LOAD FLAGS(5);          /* R_E */
0104     data PT_LOAD FLAGS(6);          /* RW_ */
0105 #ifdef CONFIG_X86_64
0106 #ifdef CONFIG_SMP
0107     percpu PT_LOAD FLAGS(6);        /* RW_ */
0108 #endif
0109     init PT_LOAD FLAGS(7);          /* RWE */
0110 #endif
0111     note PT_NOTE FLAGS(0);          /* ___ */
0112 }
0113
0114 SECTIONS
0115 {
0116 #ifdef CONFIG_X86_32
0117     . = LOAD_OFFSET + LOAD_PHYSICAL_ADDR;
0118     phys_startup_32 = ABSOLUTE(startup_32 - LOAD_OFFSET);
0119 #else
0120     . = __START_KERNEL;
0121     phys_startup_64 = ABSOLUTE(startup_64 - LOAD_OFFSET);
0122 #endif
0123
0124     /* Text and read-only data */
0125     .text :  AT(ADDR(.text) - LOAD_OFFSET) {
0126         _text = .;
0127         _stext = .;
0128         /* bootstrapping code */
0129         HEAD_TEXT
0130         TEXT_TEXT
0131         SCHED_TEXT
0132         CPUIDLE_TEXT
0133         LOCK_TEXT
0134         KPROBES_TEXT
0135         ALIGN_ENTRY_TEXT_BEGIN
0136         ENTRY_TEXT
0137         ALIGN_ENTRY_TEXT_END
0138         SOFTIRQENTRY_TEXT
0139         STATIC_CALL_TEXT
0140         *(.gnu.warning)
0141
0142 #ifdef CONFIG_RETPOLINE
0143         __indirect_thunk_start = .;
0144         *(.text.__x86.*)
0145         __indirect_thunk_end = .;
0146 #endif
0147     } :text =0xcccc
0148
0149     /* End of text section, which should occupy whole number of pages */
0150     _etext = .;
0151     . = ALIGN(PAGE_SIZE);
0152
0153     X86_ALIGN_RODATA_BEGIN
0154     RO_DATA(PAGE_SIZE)
0155     X86_ALIGN_RODATA_END
0156
0157     /* Data */
0158     .data : AT(ADDR(.data) - LOAD_OFFSET) {
0159         /* Start of data section */
0160         _sdata = .;
0161
0162         /* init_task */
0163         INIT_TASK_DATA(THREAD_SIZE)
0164
0165 #ifdef CONFIG_X86_32
0166         /* 32 bit has nosave before _edata */
0167         NOSAVE_DATA
0168 #endif
0169
0170         PAGE_ALIGNED_DATA(PAGE_SIZE)
0171
0172         CACHELINE_ALIGNED_DATA(L1_CACHE_BYTES)
0173
0174         DATA_DATA
0175         CONSTRUCTORS
0176
0177         /* rarely changed data like cpu maps */
0178         READ_MOSTLY_DATA(INTERNODE_CACHE_BYTES)
0179
0180         /* End of data section */
0181         _edata = .;
0182     } :data
0183
0184     BUG_TABLE
0185
0186     ORC_UNWIND_TABLE
0187
0188     . = ALIGN(PAGE_SIZE);
0189     __vvar_page = .;
0190
0191     .vvar : AT(ADDR(.vvar) - LOAD_OFFSET) {
0192         /* work around gold bug 13023 */
0193         __vvar_beginning_hack = .;
0194
0195         /* Place all vvars at the offsets in asm/vvar.h. */
0196 #define EMIT_VVAR(name, offset)             \
0197         . = __vvar_beginning_hack + offset; \
0198         *(.vvar_ ## name)
0199 #include <asm/vvar.h>
0200 #undef EMIT_VVAR
0201
0202         /*
0203          * Pad the rest of the page with zeros.  Otherwise the loader
0204          * can leave garbage here.
0205          */
0206         . = __vvar_beginning_hack + PAGE_SIZE;
0207     } :data
0208
0209     . = ALIGN(__vvar_page + PAGE_SIZE, PAGE_SIZE);
0210
0211     /* Init code and data - will be freed after init */
0212     . = ALIGN(PAGE_SIZE);
0213     .init.begin : AT(ADDR(.init.begin) - LOAD_OFFSET) {
0214         __init_begin = .; /* paired with __init_end */
0215     }
0216
0217 #if defined(CONFIG_X86_64) && defined(CONFIG_SMP)
0218     /*
0219      * percpu offsets are zero-based on SMP.  PERCPU_VADDR() changes the
0220      * output PHDR, so the next output section - .init.text - should
0221      * start another segment - init.
0222      */
0223     PERCPU_VADDR(INTERNODE_CACHE_BYTES, 0, :percpu)
0224     ASSERT(SIZEOF(.data..percpu) < CONFIG_PHYSICAL_START,
0225            "per-CPU data too large - increase CONFIG_PHYSICAL_START")
0226 #endif
0227
0228     INIT_TEXT_SECTION(PAGE_SIZE)
0229 #ifdef CONFIG_X86_64
0230     :init
0231 #endif
0232
0233     /*
0234      * Section for code used exclusively before alternatives are run. All
0235      * references to such code must be patched out by alternatives, normally
0236      * by using X86_FEATURE_ALWAYS CPU feature bit.
0237      *
0238      * See static_cpu_has() for an example.
0239      */
0240     .altinstr_aux : AT(ADDR(.altinstr_aux) - LOAD_OFFSET) {
0241         *(.altinstr_aux)
0242     }
0243
0244     INIT_DATA_SECTION(16)
0245
0246     .x86_cpu_dev.init : AT(ADDR(.x86_cpu_dev.init) - LOAD_OFFSET) {
0247         __x86_cpu_dev_start = .;
0248         *(.x86_cpu_dev.init)
0249         __x86_cpu_dev_end = .;
0250     }
0251
0252 #ifdef CONFIG_X86_INTEL_MID
0253     .x86_intel_mid_dev.init : AT(ADDR(.x86_intel_mid_dev.init) - \
0254                                 LOAD_OFFSET) {
0255         __x86_intel_mid_dev_start = .;
0256         *(.x86_intel_mid_dev.init)
0257         __x86_intel_mid_dev_end = .;
0258     }
0259 #endif
0260
0261     /*
0262      * start address and size of operations which during runtime
0263      * can be patched with virtualization friendly instructions or
0264      * baremetal native ones. Think page table operations.
0265      * Details in paravirt_types.h
0266      */
0267     . = ALIGN(8);
0268     .parainstructions : AT(ADDR(.parainstructions) - LOAD_OFFSET) {
0269         __parainstructions = .;
0270         *(.parainstructions)
0271         __parainstructions_end = .;
0272     }
0273
0274 #ifdef CONFIG_RETPOLINE
0275     /*
0276      * List of instructions that call/jmp/jcc to retpoline thunks
0277      * __x86_indirect_thunk_*(). These instructions can be patched along
0278      * with alternatives, after which the section can be freed.
0279      */
0280     . = ALIGN(8);
0281     .retpoline_sites : AT(ADDR(.retpoline_sites) - LOAD_OFFSET) {
0282         __retpoline_sites = .;
0283         *(.retpoline_sites)
0284         __retpoline_sites_end = .;
0285     }
0286
0287     . = ALIGN(8);
0288     .return_sites : AT(ADDR(.return_sites) - LOAD_OFFSET) {
0289         __return_sites = .;
0290         *(.return_sites)
0291         __return_sites_end = .;
0292     }
0293 #endif
0294
0295 #ifdef CONFIG_X86_KERNEL_IBT
0296     . = ALIGN(8);
0297     .ibt_endbr_seal : AT(ADDR(.ibt_endbr_seal) - LOAD_OFFSET) {
0298         __ibt_endbr_seal = .;
0299         *(.ibt_endbr_seal)
0300         __ibt_endbr_seal_end = .;
0301     }
0302 #endif
0303
0304     /*
0305      * struct alt_inst entries. From the header (alternative.h):
0306      * "Alternative instructions for different CPU types or capabilities"
0307      * Think locking instructions on spinlocks.
0308      */
0309     . = ALIGN(8);
0310     .altinstructions : AT(ADDR(.altinstructions) - LOAD_OFFSET) {
0311         __alt_instructions = .;
0312         *(.altinstructions)
0313         __alt_instructions_end = .;
0314     }
0315
0316     /*
0317      * And here are the replacement instructions. The linker sticks
0318      * them as binary blobs. The .altinstructions has enough data to
0319      * get the address and the length of them to patch the kernel safely.
0320      */
0321     .altinstr_replacement : AT(ADDR(.altinstr_replacement) - LOAD_OFFSET) {
0322         *(.altinstr_replacement)
0323     }
0324
0325     . = ALIGN(8);
0326     .apicdrivers : AT(ADDR(.apicdrivers) - LOAD_OFFSET) {
0327         __apicdrivers = .;
0328         *(.apicdrivers);
0329         __apicdrivers_end = .;
0330     }
0331
0332     . = ALIGN(8);
0333     /*
0334      * .exit.text is discarded at runtime, not link time, to deal with
0335      *  references from .altinstructions
0336      */
0337     .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) {
0338         EXIT_TEXT
0339     }
0340
0341     .exit.data : AT(ADDR(.exit.data) - LOAD_OFFSET) {
0342         EXIT_DATA
0343     }
0344
0345 #if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP)
0346     PERCPU_SECTION(INTERNODE_CACHE_BYTES)
0347 #endif
0348
0349     . = ALIGN(PAGE_SIZE);
0350
0351     /* freed after init ends here */
0352     .init.end : AT(ADDR(.init.end) - LOAD_OFFSET) {
0353         __init_end = .;
0354     }
0355
0356     /*
0357      * smp_locks might be freed after init
0358      * start/end must be page aligned
0359      */
0360     . = ALIGN(PAGE_SIZE);
0361     .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) {
0362         __smp_locks = .;
0363         *(.smp_locks)
0364         . = ALIGN(PAGE_SIZE);
0365         __smp_locks_end = .;
0366     }
0367
0368 #ifdef CONFIG_X86_64
0369     .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) {
0370         NOSAVE_DATA
0371     }
0372 #endif
0373
0374     /* BSS */
0375     . = ALIGN(PAGE_SIZE);
0376     .bss : AT(ADDR(.bss) - LOAD_OFFSET) {
0377         __bss_start = .;
0378         *(.bss..page_aligned)
0379         . = ALIGN(PAGE_SIZE);
0380         *(BSS_MAIN)
0381         BSS_DECRYPTED
0382         . = ALIGN(PAGE_SIZE);
0383         __bss_stop = .;
0384     }
0385
0386     /*
0387      * The memory occupied from _text to here, __end_of_kernel_reserve, is
0388      * automatically reserved in setup_arch(). Anything after here must be
0389      * explicitly reserved using memblock_reserve() or it will be discarded
0390      * and treated as available memory.
0391      */
0392     __end_of_kernel_reserve = .;
0393
0394     . = ALIGN(PAGE_SIZE);
0395     .brk : AT(ADDR(.brk) - LOAD_OFFSET) {
0396         __brk_base = .;
0397         . += 64 * 1024;     /* 64k alignment slop space */
0398         *(.bss..brk)        /* areas brk users have reserved */
0399         __brk_limit = .;
0400     }
0401
0402     . = ALIGN(PAGE_SIZE);       /* keep VO_INIT_SIZE page aligned */
0403     _end = .;
0404
0405 #ifdef CONFIG_AMD_MEM_ENCRYPT
0406     /*
0407      * Early scratch/workarea section: Lives outside of the kernel proper
0408      * (_text - _end).
0409      *
0410      * Resides after _end because even though the .brk section is after
0411      * __end_of_kernel_reserve, the .brk section is later reserved as a
0412      * part of the kernel. Since it is located after __end_of_kernel_reserve
0413      * it will be discarded and become part of the available memory. As
0414      * such, it can only be used by very early boot code and must not be
0415      * needed afterwards.
0416      *
0417      * Currently used by SME for performing in-place encryption of the
0418      * kernel during boot. Resides on a 2MB boundary to simplify the
0419      * pagetable setup used for SME in-place encryption.
0420      */
0421     . = ALIGN(HPAGE_SIZE);
0422     .init.scratch : AT(ADDR(.init.scratch) - LOAD_OFFSET) {
0423         __init_scratch_begin = .;
0424         *(.init.scratch)
0425         . = ALIGN(HPAGE_SIZE);
0426         __init_scratch_end = .;
0427     }
0428 #endif
0429
0430     STABS_DEBUG
0431     DWARF_DEBUG
0432     ELF_DETAILS
0433
0434     DISCARDS
0435
0436     /*
0437      * Make sure that the .got.plt is either completely empty or it
0438      * contains only the lazy dispatch entries.
0439      */
0440     .got.plt (INFO) : { *(.got.plt) }
0441     ASSERT(SIZEOF(.got.plt) == 0 ||
0442 #ifdef CONFIG_X86_64
0443            SIZEOF(.got.plt) == 0x18,
0444 #else
0445            SIZEOF(.got.plt) == 0xc,
0446 #endif
0447            "Unexpected GOT/PLT entries detected!")
0448
0449     /*
0450      * Sections that should stay zero sized, which is safer to
0451      * explicitly check instead of blindly discarding.
0452      */
0453     .got : {
0454         *(.got) *(.igot.*)
0455     }
0456     ASSERT(SIZEOF(.got) == 0, "Unexpected GOT entries detected!")
0457
0458     .plt : {
0459         *(.plt) *(.plt.*) *(.iplt)
0460     }
0461     ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!")
0462
0463     .rel.dyn : {
0464         *(.rel.*) *(.rel_*)
0465     }
0466     ASSERT(SIZEOF(.rel.dyn) == 0, "Unexpected run-time relocations (.rel) detected!")
0467
0468     .rela.dyn : {
0469         *(.rela.*) *(.rela_*)
0470     }
0471     ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!")
0472 }
0473
0474 /*
0475  * The ASSERT() sink to . is intentional, for binutils 2.14 compatibility:
0476  */
0477 . = ASSERT((_end - LOAD_OFFSET <= KERNEL_IMAGE_SIZE),
0478        "kernel image bigger than KERNEL_IMAGE_SIZE");
0479
0480 #ifdef CONFIG_X86_64
0481 /*
0482  * Per-cpu symbols which need to be offset from __per_cpu_load
0483  * for the boot processor.
0484  */
0485 #define INIT_PER_CPU(x) init_per_cpu__##x = ABSOLUTE(x) + __per_cpu_load
0486 INIT_PER_CPU(gdt_page);
0487 INIT_PER_CPU(fixed_percpu_data);
0488 INIT_PER_CPU(irq_stack_backing_store);
0489
0490 #ifdef CONFIG_SMP
0491 . = ASSERT((fixed_percpu_data == 0),
0492            "fixed_percpu_data is not at start of per-cpu area");
0493 #endif
0494
0495 #endif /* CONFIG_X86_64 */
0496
0497 #ifdef CONFIG_KEXEC_CORE
0498 #include <asm/kexec.h>
0499
0500 . = ASSERT(kexec_control_code_size <= KEXEC_CONTROL_CODE_MAX_SIZE,
0501            "kexec control code size is too big");
0502 #endif
0503