arm64/kernel/vmlinux.lds.S

0001 /* SPDX-License-Identifier: GPL-2.0 */
0002 /*
0003  * ld script to make ARM Linux kernel
0004  * taken from the i386 version by Russell King
0005  * Written by Martin Mares <mj@atrey.karlin.mff.cuni.cz>
0006  */
0007
0008 #include <asm/hyp_image.h>
0009 #ifdef CONFIG_KVM
0010 #define HYPERVISOR_EXTABLE                  \
0011     . = ALIGN(SZ_8);                    \
0012     __start___kvm_ex_table = .;             \
0013     *(__kvm_ex_table)                   \
0014     __stop___kvm_ex_table = .;
0015
0016 #define HYPERVISOR_DATA_SECTIONS                \
0017     HYP_SECTION_NAME(.rodata) : {               \
0018         . = ALIGN(PAGE_SIZE);               \
0019         __hyp_rodata_start = .;             \
0020         *(HYP_SECTION_NAME(.data..ro_after_init))   \
0021         *(HYP_SECTION_NAME(.rodata))            \
0022         . = ALIGN(PAGE_SIZE);               \
0023         __hyp_rodata_end = .;               \
0024     }
0025
0026 #define HYPERVISOR_PERCPU_SECTION               \
0027     . = ALIGN(PAGE_SIZE);                   \
0028     HYP_SECTION_NAME(.data..percpu) : {         \
0029         *(HYP_SECTION_NAME(.data..percpu))      \
0030     }
0031
0032 #define HYPERVISOR_RELOC_SECTION                \
0033     .hyp.reloc : ALIGN(4) {                 \
0034         __hyp_reloc_begin = .;              \
0035         *(.hyp.reloc)                   \
0036         __hyp_reloc_end = .;                \
0037     }
0038
0039 #define BSS_FIRST_SECTIONS                  \
0040     __hyp_bss_start = .;                    \
0041     *(HYP_SECTION_NAME(.bss))               \
0042     . = ALIGN(PAGE_SIZE);                   \
0043     __hyp_bss_end = .;
0044
0045 /*
0046  * We require that __hyp_bss_start and __bss_start are aligned, and enforce it
0047  * with an assertion. But the BSS_SECTION macro places an empty .sbss section
0048  * between them, which can in some cases cause the linker to misalign them. To
0049  * work around the issue, force a page alignment for __bss_start.
0050  */
0051 #define SBSS_ALIGN          PAGE_SIZE
0052 #else /* CONFIG_KVM */
0053 #define HYPERVISOR_EXTABLE
0054 #define HYPERVISOR_DATA_SECTIONS
0055 #define HYPERVISOR_PERCPU_SECTION
0056 #define HYPERVISOR_RELOC_SECTION
0057 #define SBSS_ALIGN          0
0058 #endif
0059
0060 #define RO_EXCEPTION_TABLE_ALIGN    4
0061 #define RUNTIME_DISCARD_EXIT
0062
0063 #include <asm-generic/vmlinux.lds.h>
0064 #include <asm/cache.h>
0065 #include <asm/kernel-pgtable.h>
0066 #include <asm/kexec.h>
0067 #include <asm/memory.h>
0068 #include <asm/page.h>
0069
0070 #include "image.h"
0071
0072 OUTPUT_ARCH(aarch64)
0073 ENTRY(_text)
0074
0075 jiffies = jiffies_64;
0076
0077 #define HYPERVISOR_TEXT                 \
0078     . = ALIGN(PAGE_SIZE);               \
0079     __hyp_idmap_text_start = .;         \
0080     *(.hyp.idmap.text)              \
0081     __hyp_idmap_text_end = .;           \
0082     __hyp_text_start = .;               \
0083     *(.hyp.text)                    \
0084     HYPERVISOR_EXTABLE              \
0085     . = ALIGN(PAGE_SIZE);               \
0086     __hyp_text_end = .;
0087
0088 #define IDMAP_TEXT                  \
0089     . = ALIGN(SZ_4K);               \
0090     __idmap_text_start = .;             \
0091     *(.idmap.text)                  \
0092     __idmap_text_end = .;
0093
0094 #ifdef CONFIG_HIBERNATION
0095 #define HIBERNATE_TEXT                  \
0096     __hibernate_exit_text_start = .;        \
0097     *(.hibernate_exit.text)             \
0098     __hibernate_exit_text_end = .;
0099 #else
0100 #define HIBERNATE_TEXT
0101 #endif
0102
0103 #ifdef CONFIG_KEXEC_CORE
0104 #define KEXEC_TEXT                  \
0105     __relocate_new_kernel_start = .;        \
0106     *(.kexec_relocate.text)             \
0107     __relocate_new_kernel_end = .;
0108 #else
0109 #define KEXEC_TEXT
0110 #endif
0111
0112 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
0113 #define TRAMP_TEXT                  \
0114     . = ALIGN(PAGE_SIZE);               \
0115     __entry_tramp_text_start = .;           \
0116     *(.entry.tramp.text)                \
0117     . = ALIGN(PAGE_SIZE);               \
0118     __entry_tramp_text_end = .;         \
0119     *(.entry.tramp.rodata)
0120 #else
0121 #define TRAMP_TEXT
0122 #endif
0123
0124 /*
0125  * The size of the PE/COFF section that covers the kernel image, which
0126  * runs from _stext to _edata, must be a round multiple of the PE/COFF
0127  * FileAlignment, which we set to its minimum value of 0x200. '_stext'
0128  * itself is 4 KB aligned, so padding out _edata to a 0x200 aligned
0129  * boundary should be sufficient.
0130  */
0131 PECOFF_FILE_ALIGNMENT = 0x200;
0132
0133 #ifdef CONFIG_EFI
0134 #define PECOFF_EDATA_PADDING    \
0135     .pecoff_edata_padding : { BYTE(0); . = ALIGN(PECOFF_FILE_ALIGNMENT); }
0136 #else
0137 #define PECOFF_EDATA_PADDING
0138 #endif
0139
0140 SECTIONS
0141 {
0142     /*
0143      * XXX: The linker does not define how output sections are
0144      * assigned to input sections when there are multiple statements
0145      * matching the same input section name.  There is no documented
0146      * order of matching.
0147      */
0148     DISCARDS
0149     /DISCARD/ : {
0150         *(.interp .dynamic)
0151         *(.dynsym .dynstr .hash .gnu.hash)
0152     }
0153
0154     . = KIMAGE_VADDR;
0155
0156     .head.text : {
0157         _text = .;
0158         HEAD_TEXT
0159     }
0160     .text : ALIGN(SEGMENT_ALIGN) {  /* Real text segment        */
0161         _stext = .;     /* Text and read-only data  */
0162             IRQENTRY_TEXT
0163             SOFTIRQENTRY_TEXT
0164             ENTRY_TEXT
0165             TEXT_TEXT
0166             SCHED_TEXT
0167             CPUIDLE_TEXT
0168             LOCK_TEXT
0169             KPROBES_TEXT
0170             HYPERVISOR_TEXT
0171             IDMAP_TEXT
0172             *(.gnu.warning)
0173         . = ALIGN(16);
0174         *(.got)         /* Global offset table      */
0175     }
0176
0177     /*
0178      * Make sure that the .got.plt is either completely empty or it
0179      * contains only the lazy dispatch entries.
0180      */
0181     .got.plt : { *(.got.plt) }
0182     ASSERT(SIZEOF(.got.plt) == 0 || SIZEOF(.got.plt) == 0x18,
0183            "Unexpected GOT/PLT entries detected!")
0184
0185     . = ALIGN(SEGMENT_ALIGN);
0186     _etext = .;         /* End of text section */
0187
0188     /* everything from this point to __init_begin will be marked RO NX */
0189     RO_DATA(PAGE_SIZE)
0190
0191     HYPERVISOR_DATA_SECTIONS
0192
0193     /* code sections that are never executed via the kernel mapping */
0194     .rodata.text : {
0195         TRAMP_TEXT
0196         HIBERNATE_TEXT
0197         KEXEC_TEXT
0198         . = ALIGN(PAGE_SIZE);
0199     }
0200
0201     idmap_pg_dir = .;
0202     . += PAGE_SIZE;
0203
0204 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
0205     tramp_pg_dir = .;
0206     . += PAGE_SIZE;
0207 #endif
0208
0209     reserved_pg_dir = .;
0210     . += PAGE_SIZE;
0211
0212     swapper_pg_dir = .;
0213     . += PAGE_SIZE;
0214
0215     . = ALIGN(SEGMENT_ALIGN);
0216     __init_begin = .;
0217     __inittext_begin = .;
0218
0219     INIT_TEXT_SECTION(8)
0220
0221     __exittext_begin = .;
0222     .exit.text : {
0223         EXIT_TEXT
0224     }
0225     __exittext_end = .;
0226
0227     . = ALIGN(4);
0228     .altinstructions : {
0229         __alt_instructions = .;
0230         *(.altinstructions)
0231         __alt_instructions_end = .;
0232     }
0233
0234     . = ALIGN(SEGMENT_ALIGN);
0235     __inittext_end = .;
0236     __initdata_begin = .;
0237
0238     init_idmap_pg_dir = .;
0239     . += INIT_IDMAP_DIR_SIZE;
0240     init_idmap_pg_end = .;
0241
0242     .init.data : {
0243         INIT_DATA
0244         INIT_SETUP(16)
0245         INIT_CALLS
0246         CON_INITCALL
0247         INIT_RAM_FS
0248         *(.init.altinstructions .init.bss)  /* from the EFI stub */
0249     }
0250     .exit.data : {
0251         EXIT_DATA
0252     }
0253
0254     PERCPU_SECTION(L1_CACHE_BYTES)
0255     HYPERVISOR_PERCPU_SECTION
0256
0257     HYPERVISOR_RELOC_SECTION
0258
0259     .rela.dyn : ALIGN(8) {
0260         __rela_start = .;
0261         *(.rela .rela*)
0262         __rela_end = .;
0263     }
0264
0265     .relr.dyn : ALIGN(8) {
0266         __relr_start = .;
0267         *(.relr.dyn)
0268         __relr_end = .;
0269     }
0270
0271     . = ALIGN(SEGMENT_ALIGN);
0272     __initdata_end = .;
0273     __init_end = .;
0274
0275     _data = .;
0276     _sdata = .;
0277     RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
0278
0279     /*
0280      * Data written with the MMU off but read with the MMU on requires
0281      * cache lines to be invalidated, discarding up to a Cache Writeback
0282      * Granule (CWG) of data from the cache. Keep the section that
0283      * requires this type of maintenance to be in its own Cache Writeback
0284      * Granule (CWG) area so the cache maintenance operations don't
0285      * interfere with adjacent data.
0286      */
0287     .mmuoff.data.write : ALIGN(SZ_2K) {
0288         __mmuoff_data_start = .;
0289         *(.mmuoff.data.write)
0290     }
0291     . = ALIGN(SZ_2K);
0292     .mmuoff.data.read : {
0293         *(.mmuoff.data.read)
0294         __mmuoff_data_end = .;
0295     }
0296
0297     PECOFF_EDATA_PADDING
0298     __pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin);
0299     _edata = .;
0300
0301     BSS_SECTION(SBSS_ALIGN, 0, 0)
0302
0303     . = ALIGN(PAGE_SIZE);
0304     init_pg_dir = .;
0305     . += INIT_DIR_SIZE;
0306     init_pg_end = .;
0307
0308     . = ALIGN(SEGMENT_ALIGN);
0309     __pecoff_data_size = ABSOLUTE(. - __initdata_begin);
0310     _end = .;
0311
0312     STABS_DEBUG
0313     DWARF_DEBUG
0314     ELF_DETAILS
0315
0316     HEAD_SYMBOLS
0317
0318     /*
0319      * Sections that should stay zero sized, which is safer to
0320      * explicitly check instead of blindly discarding.
0321      */
0322     .plt : {
0323         *(.plt) *(.plt.*) *(.iplt) *(.igot .igot.plt)
0324     }
0325     ASSERT(SIZEOF(.plt) == 0, "Unexpected run-time procedure linkages detected!")
0326
0327     .data.rel.ro : { *(.data.rel.ro) }
0328     ASSERT(SIZEOF(.data.rel.ro) == 0, "Unexpected RELRO detected!")
0329 }
0330
0331 #include "image-vars.h"
0332
0333 /*
0334  * The HYP init code and ID map text can't be longer than a page each. The
0335  * former is page-aligned, but the latter may not be with 16K or 64K pages, so
0336  * it should also not cross a page boundary.
0337  */
0338 ASSERT(__hyp_idmap_text_end - __hyp_idmap_text_start <= PAGE_SIZE,
0339     "HYP init code too big")
0340 ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
0341     "ID map text too big or misaligned")
0342 #ifdef CONFIG_HIBERNATION
0343 ASSERT(__hibernate_exit_text_end - __hibernate_exit_text_start <= SZ_4K,
0344        "Hibernate exit text is bigger than 4 KiB")
0345 #endif
0346 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
0347 ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) <= 3*PAGE_SIZE,
0348     "Entry trampoline text too big")
0349 #endif
0350 #ifdef CONFIG_KVM
0351 ASSERT(__hyp_bss_start == __bss_start, "HYP and Host BSS are misaligned")
0352 #endif
0353 /*
0354  * If padding is applied before .head.text, virt<->phys conversions will fail.
0355  */
0356 ASSERT(_text == KIMAGE_VADDR, "HEAD is misaligned")
0357
0358 ASSERT(swapper_pg_dir - reserved_pg_dir == RESERVED_SWAPPER_OFFSET,
0359        "RESERVED_SWAPPER_OFFSET is wrong!")
0360
0361 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
0362 ASSERT(swapper_pg_dir - tramp_pg_dir == TRAMP_SWAPPER_OFFSET,
0363        "TRAMP_SWAPPER_OFFSET is wrong!")
0364 #endif
0365
0366 #ifdef CONFIG_KEXEC_CORE
0367 /* kexec relocation code should fit into one KEXEC_CONTROL_PAGE_SIZE */
0368 ASSERT(__relocate_new_kernel_end - __relocate_new_kernel_start <= SZ_4K,
0369        "kexec relocation code is bigger than 4 KiB")
0370 ASSERT(KEXEC_CONTROL_PAGE_SIZE >= SZ_4K, "KEXEC_CONTROL_PAGE_SIZE is broken")
0371 #endif