x86/kernel/relocate_kernel_64.S

0001 /* SPDX-License-Identifier: GPL-2.0-only */
0002 /*
0003  * relocate_kernel.S - put the kernel image in place to boot
0004  * Copyright (C) 2002-2005 Eric Biederman  <ebiederm@xmission.com>
0005  */
0006
0007 #include <linux/linkage.h>
0008 #include <asm/page_types.h>
0009 #include <asm/kexec.h>
0010 #include <asm/processor-flags.h>
0011 #include <asm/pgtable_types.h>
0012 #include <asm/nospec-branch.h>
0013 #include <asm/unwind_hints.h>
0014
0015 /*
0016  * Must be relocatable PIC code callable as a C function, in particular
0017  * there must be a plain RET and not jump to return thunk.
0018  */
0019
0020 #define PTR(x) (x << 3)
0021 #define PAGE_ATTR (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY)
0022
0023 /*
0024  * control_page + KEXEC_CONTROL_CODE_MAX_SIZE
0025  * ~ control_page + PAGE_SIZE are used as data storage and stack for
0026  * jumping back
0027  */
0028 #define DATA(offset)        (KEXEC_CONTROL_CODE_MAX_SIZE+(offset))
0029
0030 /* Minimal CPU state */
0031 #define RSP         DATA(0x0)
0032 #define CR0         DATA(0x8)
0033 #define CR3         DATA(0x10)
0034 #define CR4         DATA(0x18)
0035
0036 /* other data */
0037 #define CP_PA_TABLE_PAGE    DATA(0x20)
0038 #define CP_PA_SWAP_PAGE     DATA(0x28)
0039 #define CP_PA_BACKUP_PAGES_MAP  DATA(0x30)
0040
0041     .text
0042     .align PAGE_SIZE
0043     .code64
0044 SYM_CODE_START_NOALIGN(relocate_kernel)
0045     UNWIND_HINT_EMPTY
0046     ANNOTATE_NOENDBR
0047     /*
0048      * %rdi indirection_page
0049      * %rsi page_list
0050      * %rdx start address
0051      * %rcx preserve_context
0052      * %r8  host_mem_enc_active
0053      */
0054
0055     /* Save the CPU context, used for jumping back */
0056     pushq %rbx
0057     pushq %rbp
0058     pushq %r12
0059     pushq %r13
0060     pushq %r14
0061     pushq %r15
0062     pushf
0063
0064     movq    PTR(VA_CONTROL_PAGE)(%rsi), %r11
0065     movq    %rsp, RSP(%r11)
0066     movq    %cr0, %rax
0067     movq    %rax, CR0(%r11)
0068     movq    %cr3, %rax
0069     movq    %rax, CR3(%r11)
0070     movq    %cr4, %rax
0071     movq    %rax, CR4(%r11)
0072
0073     /* Save CR4. Required to enable the right paging mode later. */
0074     movq    %rax, %r13
0075
0076     /* zero out flags, and disable interrupts */
0077     pushq $0
0078     popfq
0079
0080     /* Save SME active flag */
0081     movq    %r8, %r12
0082
0083     /*
0084      * get physical address of control page now
0085      * this is impossible after page table switch
0086      */
0087     movq    PTR(PA_CONTROL_PAGE)(%rsi), %r8
0088
0089     /* get physical address of page table now too */
0090     movq    PTR(PA_TABLE_PAGE)(%rsi), %r9
0091
0092     /* get physical address of swap page now */
0093     movq    PTR(PA_SWAP_PAGE)(%rsi), %r10
0094
0095     /* save some information for jumping back */
0096     movq    %r9, CP_PA_TABLE_PAGE(%r11)
0097     movq    %r10, CP_PA_SWAP_PAGE(%r11)
0098     movq    %rdi, CP_PA_BACKUP_PAGES_MAP(%r11)
0099
0100     /* Switch to the identity mapped page tables */
0101     movq    %r9, %cr3
0102
0103     /* setup a new stack at the end of the physical control page */
0104     lea PAGE_SIZE(%r8), %rsp
0105
0106     /* jump to identity mapped page */
0107     addq    $(identity_mapped - relocate_kernel), %r8
0108     pushq   %r8
0109     ANNOTATE_UNRET_SAFE
0110     ret
0111     int3
0112 SYM_CODE_END(relocate_kernel)
0113
0114 SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
0115     UNWIND_HINT_EMPTY
0116     /* set return address to 0 if not preserving context */
0117     pushq   $0
0118     /* store the start address on the stack */
0119     pushq   %rdx
0120
0121     /*
0122      * Clear X86_CR4_CET (if it was set) such that we can clear CR0_WP
0123      * below.
0124      */
0125     movq    %cr4, %rax
0126     andq    $~(X86_CR4_CET), %rax
0127     movq    %rax, %cr4
0128
0129     /*
0130      * Set cr0 to a known state:
0131      *  - Paging enabled
0132      *  - Alignment check disabled
0133      *  - Write protect disabled
0134      *  - No task switch
0135      *  - Don't do FP software emulation.
0136      *  - Protected mode enabled
0137      */
0138     movq    %cr0, %rax
0139     andq    $~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %rax
0140     orl $(X86_CR0_PG | X86_CR0_PE), %eax
0141     movq    %rax, %cr0
0142
0143     /*
0144      * Set cr4 to a known state:
0145      *  - physical address extension enabled
0146      *  - 5-level paging, if it was enabled before
0147      */
0148     movl    $X86_CR4_PAE, %eax
0149     testq   $X86_CR4_LA57, %r13
0150     jz  1f
0151     orl $X86_CR4_LA57, %eax
0152 1:
0153     movq    %rax, %cr4
0154
0155     jmp 1f
0156 1:
0157
0158     /* Flush the TLB (needed?) */
0159     movq    %r9, %cr3
0160
0161     /*
0162      * If SME is active, there could be old encrypted cache line
0163      * entries that will conflict with the now unencrypted memory
0164      * used by kexec. Flush the caches before copying the kernel.
0165      */
0166     testq   %r12, %r12
0167     jz 1f
0168     wbinvd
0169 1:
0170
0171     movq    %rcx, %r11
0172     call    swap_pages
0173
0174     /*
0175      * To be certain of avoiding problems with self-modifying code
0176      * I need to execute a serializing instruction here.
0177      * So I flush the TLB by reloading %cr3 here, it's handy,
0178      * and not processor dependent.
0179      */
0180     movq    %cr3, %rax
0181     movq    %rax, %cr3
0182
0183     /*
0184      * set all of the registers to known values
0185      * leave %rsp alone
0186      */
0187
0188     testq   %r11, %r11
0189     jnz 1f
0190     xorl    %eax, %eax
0191     xorl    %ebx, %ebx
0192     xorl    %ecx, %ecx
0193     xorl    %edx, %edx
0194     xorl    %esi, %esi
0195     xorl    %edi, %edi
0196     xorl    %ebp, %ebp
0197     xorl    %r8d, %r8d
0198     xorl    %r9d, %r9d
0199     xorl    %r10d, %r10d
0200     xorl    %r11d, %r11d
0201     xorl    %r12d, %r12d
0202     xorl    %r13d, %r13d
0203     xorl    %r14d, %r14d
0204     xorl    %r15d, %r15d
0205
0206     ANNOTATE_UNRET_SAFE
0207     ret
0208     int3
0209
0210 1:
0211     popq    %rdx
0212     leaq    PAGE_SIZE(%r10), %rsp
0213     ANNOTATE_RETPOLINE_SAFE
0214     call    *%rdx
0215
0216     /* get the re-entry point of the peer system */
0217     movq    0(%rsp), %rbp
0218     leaq    relocate_kernel(%rip), %r8
0219     movq    CP_PA_SWAP_PAGE(%r8), %r10
0220     movq    CP_PA_BACKUP_PAGES_MAP(%r8), %rdi
0221     movq    CP_PA_TABLE_PAGE(%r8), %rax
0222     movq    %rax, %cr3
0223     lea PAGE_SIZE(%r8), %rsp
0224     call    swap_pages
0225     movq    $virtual_mapped, %rax
0226     pushq   %rax
0227     ANNOTATE_UNRET_SAFE
0228     ret
0229     int3
0230 SYM_CODE_END(identity_mapped)
0231
0232 SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped)
0233     UNWIND_HINT_EMPTY
0234     ANNOTATE_NOENDBR // RET target, above
0235     movq    RSP(%r8), %rsp
0236     movq    CR4(%r8), %rax
0237     movq    %rax, %cr4
0238     movq    CR3(%r8), %rax
0239     movq    CR0(%r8), %r8
0240     movq    %rax, %cr3
0241     movq    %r8, %cr0
0242     movq    %rbp, %rax
0243
0244     popf
0245     popq    %r15
0246     popq    %r14
0247     popq    %r13
0248     popq    %r12
0249     popq    %rbp
0250     popq    %rbx
0251     ANNOTATE_UNRET_SAFE
0252     ret
0253     int3
0254 SYM_CODE_END(virtual_mapped)
0255
0256     /* Do the copies */
0257 SYM_CODE_START_LOCAL_NOALIGN(swap_pages)
0258     UNWIND_HINT_EMPTY
0259     movq    %rdi, %rcx  /* Put the page_list in %rcx */
0260     xorl    %edi, %edi
0261     xorl    %esi, %esi
0262     jmp 1f
0263
0264 0:  /* top, read another word for the indirection page */
0265
0266     movq    (%rbx), %rcx
0267     addq    $8, %rbx
0268 1:
0269     testb   $0x1,   %cl   /* is it a destination page? */
0270     jz  2f
0271     movq    %rcx,   %rdi
0272     andq    $0xfffffffffffff000, %rdi
0273     jmp 0b
0274 2:
0275     testb   $0x2,   %cl   /* is it an indirection page? */
0276     jz  2f
0277     movq    %rcx,   %rbx
0278     andq    $0xfffffffffffff000, %rbx
0279     jmp 0b
0280 2:
0281     testb   $0x4,   %cl   /* is it the done indicator? */
0282     jz  2f
0283     jmp 3f
0284 2:
0285     testb   $0x8,   %cl   /* is it the source indicator? */
0286     jz  0b        /* Ignore it otherwise */
0287     movq    %rcx,   %rsi  /* For ever source page do a copy */
0288     andq    $0xfffffffffffff000, %rsi
0289
0290     movq    %rdi, %rdx
0291     movq    %rsi, %rax
0292
0293     movq    %r10, %rdi
0294     movl    $512, %ecx
0295     rep ; movsq
0296
0297     movq    %rax, %rdi
0298     movq    %rdx, %rsi
0299     movl    $512, %ecx
0300     rep ; movsq
0301
0302     movq    %rdx, %rdi
0303     movq    %r10, %rsi
0304     movl    $512, %ecx
0305     rep ; movsq
0306
0307     lea PAGE_SIZE(%rax), %rsi
0308     jmp 0b
0309 3:
0310     ANNOTATE_UNRET_SAFE
0311     ret
0312     int3
0313 SYM_CODE_END(swap_pages)
0314
0315     .globl kexec_control_code_size
0316 .set kexec_control_code_size, . - relocate_kernel