OSCL-LXR |
|
|
|
|||
0001 /* SPDX-License-Identifier: GPL-2.0-only */ 0002 /* 0003 * AMD Memory Encryption Support 0004 * 0005 * Copyright (C) 2016 Advanced Micro Devices, Inc. 0006 * 0007 * Author: Tom Lendacky <thomas.lendacky@amd.com> 0008 */ 0009 0010 #include <linux/linkage.h> 0011 #include <linux/pgtable.h> 0012 #include <asm/page.h> 0013 #include <asm/processor-flags.h> 0014 #include <asm/msr-index.h> 0015 #include <asm/nospec-branch.h> 0016 0017 .text 0018 .code64 0019 SYM_FUNC_START(sme_encrypt_execute) 0020 0021 /* 0022 * Entry parameters: 0023 * RDI - virtual address for the encrypted mapping 0024 * RSI - virtual address for the decrypted mapping 0025 * RDX - length to encrypt 0026 * RCX - virtual address of the encryption workarea, including: 0027 * - stack page (PAGE_SIZE) 0028 * - encryption routine page (PAGE_SIZE) 0029 * - intermediate copy buffer (PMD_PAGE_SIZE) 0030 * R8 - physical address of the pagetables to use for encryption 0031 */ 0032 0033 push %rbp 0034 movq %rsp, %rbp /* RBP now has original stack pointer */ 0035 0036 /* Set up a one page stack in the non-encrypted memory area */ 0037 movq %rcx, %rax /* Workarea stack page */ 0038 leaq PAGE_SIZE(%rax), %rsp /* Set new stack pointer */ 0039 addq $PAGE_SIZE, %rax /* Workarea encryption routine */ 0040 0041 push %r12 0042 movq %rdi, %r10 /* Encrypted area */ 0043 movq %rsi, %r11 /* Decrypted area */ 0044 movq %rdx, %r12 /* Area length */ 0045 0046 /* Copy encryption routine into the workarea */ 0047 movq %rax, %rdi /* Workarea encryption routine */ 0048 leaq __enc_copy(%rip), %rsi /* Encryption routine */ 0049 movq $(.L__enc_copy_end - __enc_copy), %rcx /* Encryption routine length */ 0050 rep movsb 0051 0052 /* Setup registers for call */ 0053 movq %r10, %rdi /* Encrypted area */ 0054 movq %r11, %rsi /* Decrypted area */ 0055 movq %r8, %rdx /* Pagetables used for encryption */ 0056 movq %r12, %rcx /* Area length */ 0057 movq %rax, %r8 /* Workarea encryption routine */ 0058 addq $PAGE_SIZE, %r8 /* Workarea intermediate copy buffer */ 0059 0060 ANNOTATE_RETPOLINE_SAFE 0061 call *%rax /* Call the encryption routine */ 0062 0063 pop %r12 0064 0065 movq %rbp, %rsp /* Restore original stack pointer */ 0066 pop %rbp 0067 0068 /* Offset to __x86_return_thunk would be wrong here */ 0069 ANNOTATE_UNRET_SAFE 0070 ret 0071 int3 0072 SYM_FUNC_END(sme_encrypt_execute) 0073 0074 SYM_FUNC_START(__enc_copy) 0075 /* 0076 * Routine used to encrypt memory in place. 0077 * This routine must be run outside of the kernel proper since 0078 * the kernel will be encrypted during the process. So this 0079 * routine is defined here and then copied to an area outside 0080 * of the kernel where it will remain and run decrypted 0081 * during execution. 0082 * 0083 * On entry the registers must be: 0084 * RDI - virtual address for the encrypted mapping 0085 * RSI - virtual address for the decrypted mapping 0086 * RDX - address of the pagetables to use for encryption 0087 * RCX - length of area 0088 * R8 - intermediate copy buffer 0089 * 0090 * RAX - points to this routine 0091 * 0092 * The area will be encrypted by copying from the non-encrypted 0093 * memory space to an intermediate buffer and then copying from the 0094 * intermediate buffer back to the encrypted memory space. The physical 0095 * addresses of the two mappings are the same which results in the area 0096 * being encrypted "in place". 0097 */ 0098 /* Enable the new page tables */ 0099 mov %rdx, %cr3 0100 0101 /* Flush any global TLBs */ 0102 mov %cr4, %rdx 0103 andq $~X86_CR4_PGE, %rdx 0104 mov %rdx, %cr4 0105 orq $X86_CR4_PGE, %rdx 0106 mov %rdx, %cr4 0107 0108 push %r15 0109 push %r12 0110 0111 movq %rcx, %r9 /* Save area length */ 0112 movq %rdi, %r10 /* Save encrypted area address */ 0113 movq %rsi, %r11 /* Save decrypted area address */ 0114 0115 /* Set the PAT register PA5 entry to write-protect */ 0116 movl $MSR_IA32_CR_PAT, %ecx 0117 rdmsr 0118 mov %rdx, %r15 /* Save original PAT value */ 0119 andl $0xffff00ff, %edx /* Clear PA5 */ 0120 orl $0x00000500, %edx /* Set PA5 to WP */ 0121 wrmsr 0122 0123 wbinvd /* Invalidate any cache entries */ 0124 0125 /* Copy/encrypt up to 2MB at a time */ 0126 movq $PMD_PAGE_SIZE, %r12 0127 1: 0128 cmpq %r12, %r9 0129 jnb 2f 0130 movq %r9, %r12 0131 0132 2: 0133 movq %r11, %rsi /* Source - decrypted area */ 0134 movq %r8, %rdi /* Dest - intermediate copy buffer */ 0135 movq %r12, %rcx 0136 rep movsb 0137 0138 movq %r8, %rsi /* Source - intermediate copy buffer */ 0139 movq %r10, %rdi /* Dest - encrypted area */ 0140 movq %r12, %rcx 0141 rep movsb 0142 0143 addq %r12, %r11 0144 addq %r12, %r10 0145 subq %r12, %r9 /* Kernel length decrement */ 0146 jnz 1b /* Kernel length not zero? */ 0147 0148 /* Restore PAT register */ 0149 movl $MSR_IA32_CR_PAT, %ecx 0150 rdmsr 0151 mov %r15, %rdx /* Restore original PAT value */ 0152 wrmsr 0153 0154 pop %r12 0155 pop %r15 0156 0157 /* Offset to __x86_return_thunk would be wrong here */ 0158 ANNOTATE_UNRET_SAFE 0159 ret 0160 int3 0161 .L__enc_copy_end: 0162 SYM_FUNC_END(__enc_copy)
| [ Source navigation ] | [ Diff markup ] | [ Identifier search ] | [ general search ] |
|
This page was automatically generated by the 2.1.0 LXR engine. The LXR team |
|