boot/compressed/sev.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * AMD Encrypted Register State Support
0004  *
0005  * Author: Joerg Roedel <jroedel@suse.de>
0006  */
0007
0008 /*
0009  * misc.h needs to be first because it knows how to include the other kernel
0010  * headers in the pre-decompression code in a way that does not break
0011  * compilation.
0012  */
0013 #include "misc.h"
0014
0015 #include <asm/pgtable_types.h>
0016 #include <asm/sev.h>
0017 #include <asm/trapnr.h>
0018 #include <asm/trap_pf.h>
0019 #include <asm/msr-index.h>
0020 #include <asm/fpu/xcr.h>
0021 #include <asm/ptrace.h>
0022 #include <asm/svm.h>
0023 #include <asm/cpuid.h>
0024
0025 #include "error.h"
0026 #include "../msr.h"
0027
0028 struct ghcb boot_ghcb_page __aligned(PAGE_SIZE);
0029 struct ghcb *boot_ghcb;
0030
0031 /*
0032  * Copy a version of this function here - insn-eval.c can't be used in
0033  * pre-decompression code.
0034  */
0035 static bool insn_has_rep_prefix(struct insn *insn)
0036 {
0037     insn_byte_t p;
0038     int i;
0039
0040     insn_get_prefixes(insn);
0041
0042     for_each_insn_prefix(insn, i, p) {
0043         if (p == 0xf2 || p == 0xf3)
0044             return true;
0045     }
0046
0047     return false;
0048 }
0049
0050 /*
0051  * Only a dummy for insn_get_seg_base() - Early boot-code is 64bit only and
0052  * doesn't use segments.
0053  */
0054 static unsigned long insn_get_seg_base(struct pt_regs *regs, int seg_reg_idx)
0055 {
0056     return 0UL;
0057 }
0058
0059 static inline u64 sev_es_rd_ghcb_msr(void)
0060 {
0061     struct msr m;
0062
0063     boot_rdmsr(MSR_AMD64_SEV_ES_GHCB, &m);
0064
0065     return m.q;
0066 }
0067
0068 static inline void sev_es_wr_ghcb_msr(u64 val)
0069 {
0070     struct msr m;
0071
0072     m.q = val;
0073     boot_wrmsr(MSR_AMD64_SEV_ES_GHCB, &m);
0074 }
0075
0076 static enum es_result vc_decode_insn(struct es_em_ctxt *ctxt)
0077 {
0078     char buffer[MAX_INSN_SIZE];
0079     int ret;
0080
0081     memcpy(buffer, (unsigned char *)ctxt->regs->ip, MAX_INSN_SIZE);
0082
0083     ret = insn_decode(&ctxt->insn, buffer, MAX_INSN_SIZE, INSN_MODE_64);
0084     if (ret < 0)
0085         return ES_DECODE_FAILED;
0086
0087     return ES_OK;
0088 }
0089
0090 static enum es_result vc_write_mem(struct es_em_ctxt *ctxt,
0091                    void *dst, char *buf, size_t size)
0092 {
0093     memcpy(dst, buf, size);
0094
0095     return ES_OK;
0096 }
0097
0098 static enum es_result vc_read_mem(struct es_em_ctxt *ctxt,
0099                   void *src, char *buf, size_t size)
0100 {
0101     memcpy(buf, src, size);
0102
0103     return ES_OK;
0104 }
0105
0106 #undef __init
0107 #undef __pa
0108 #define __init
0109 #define __pa(x) ((unsigned long)(x))
0110
0111 #define __BOOT_COMPRESSED
0112
0113 /* Basic instruction decoding support needed */
0114 #include "../../lib/inat.c"
0115 #include "../../lib/insn.c"
0116
0117 /* Include code for early handlers */
0118 #include "../../kernel/sev-shared.c"
0119
0120 static inline bool sev_snp_enabled(void)
0121 {
0122     return sev_status & MSR_AMD64_SEV_SNP_ENABLED;
0123 }
0124
0125 static void __page_state_change(unsigned long paddr, enum psc_op op)
0126 {
0127     u64 val;
0128
0129     if (!sev_snp_enabled())
0130         return;
0131
0132     /*
0133      * If private -> shared then invalidate the page before requesting the
0134      * state change in the RMP table.
0135      */
0136     if (op == SNP_PAGE_STATE_SHARED && pvalidate(paddr, RMP_PG_SIZE_4K, 0))
0137         sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
0138
0139     /* Issue VMGEXIT to change the page state in RMP table. */
0140     sev_es_wr_ghcb_msr(GHCB_MSR_PSC_REQ_GFN(paddr >> PAGE_SHIFT, op));
0141     VMGEXIT();
0142
0143     /* Read the response of the VMGEXIT. */
0144     val = sev_es_rd_ghcb_msr();
0145     if ((GHCB_RESP_CODE(val) != GHCB_MSR_PSC_RESP) || GHCB_MSR_PSC_RESP_VAL(val))
0146         sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
0147
0148     /*
0149      * Now that page state is changed in the RMP table, validate it so that it is
0150      * consistent with the RMP entry.
0151      */
0152     if (op == SNP_PAGE_STATE_PRIVATE && pvalidate(paddr, RMP_PG_SIZE_4K, 1))
0153         sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
0154 }
0155
0156 void snp_set_page_private(unsigned long paddr)
0157 {
0158     __page_state_change(paddr, SNP_PAGE_STATE_PRIVATE);
0159 }
0160
0161 void snp_set_page_shared(unsigned long paddr)
0162 {
0163     __page_state_change(paddr, SNP_PAGE_STATE_SHARED);
0164 }
0165
0166 static bool early_setup_ghcb(void)
0167 {
0168     if (set_page_decrypted((unsigned long)&boot_ghcb_page))
0169         return false;
0170
0171     /* Page is now mapped decrypted, clear it */
0172     memset(&boot_ghcb_page, 0, sizeof(boot_ghcb_page));
0173
0174     boot_ghcb = &boot_ghcb_page;
0175
0176     /* Initialize lookup tables for the instruction decoder */
0177     inat_init_tables();
0178
0179     /* SNP guest requires the GHCB GPA must be registered */
0180     if (sev_snp_enabled())
0181         snp_register_ghcb_early(__pa(&boot_ghcb_page));
0182
0183     return true;
0184 }
0185
0186 void sev_es_shutdown_ghcb(void)
0187 {
0188     if (!boot_ghcb)
0189         return;
0190
0191     if (!sev_es_check_cpu_features())
0192         error("SEV-ES CPU Features missing.");
0193
0194     /*
0195      * GHCB Page must be flushed from the cache and mapped encrypted again.
0196      * Otherwise the running kernel will see strange cache effects when
0197      * trying to use that page.
0198      */
0199     if (set_page_encrypted((unsigned long)&boot_ghcb_page))
0200         error("Can't map GHCB page encrypted");
0201
0202     /*
0203      * GHCB page is mapped encrypted again and flushed from the cache.
0204      * Mark it non-present now to catch bugs when #VC exceptions trigger
0205      * after this point.
0206      */
0207     if (set_page_non_present((unsigned long)&boot_ghcb_page))
0208         error("Can't unmap GHCB page");
0209 }
0210
0211 bool sev_es_check_ghcb_fault(unsigned long address)
0212 {
0213     /* Check whether the fault was on the GHCB page */
0214     return ((address & PAGE_MASK) == (unsigned long)&boot_ghcb_page);
0215 }
0216
0217 void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
0218 {
0219     struct es_em_ctxt ctxt;
0220     enum es_result result;
0221
0222     if (!boot_ghcb && !early_setup_ghcb())
0223         sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
0224
0225     vc_ghcb_invalidate(boot_ghcb);
0226     result = vc_init_em_ctxt(&ctxt, regs, exit_code);
0227     if (result != ES_OK)
0228         goto finish;
0229
0230     switch (exit_code) {
0231     case SVM_EXIT_RDTSC:
0232     case SVM_EXIT_RDTSCP:
0233         result = vc_handle_rdtsc(boot_ghcb, &ctxt, exit_code);
0234         break;
0235     case SVM_EXIT_IOIO:
0236         result = vc_handle_ioio(boot_ghcb, &ctxt);
0237         break;
0238     case SVM_EXIT_CPUID:
0239         result = vc_handle_cpuid(boot_ghcb, &ctxt);
0240         break;
0241     default:
0242         result = ES_UNSUPPORTED;
0243         break;
0244     }
0245
0246 finish:
0247     if (result == ES_OK)
0248         vc_finish_insn(&ctxt);
0249     else if (result != ES_RETRY)
0250         sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
0251 }
0252
0253 static void enforce_vmpl0(void)
0254 {
0255     u64 attrs;
0256     int err;
0257
0258     /*
0259      * RMPADJUST modifies RMP permissions of a lesser-privileged (numerically
0260      * higher) privilege level. Here, clear the VMPL1 permission mask of the
0261      * GHCB page. If the guest is not running at VMPL0, this will fail.
0262      *
0263      * If the guest is running at VMPL0, it will succeed. Even if that operation
0264      * modifies permission bits, it is still ok to do so currently because Linux
0265      * SNP guests are supported only on VMPL0 so VMPL1 or higher permission masks
0266      * changing is a don't-care.
0267      */
0268     attrs = 1;
0269     if (rmpadjust((unsigned long)&boot_ghcb_page, RMP_PG_SIZE_4K, attrs))
0270         sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NOT_VMPL0);
0271 }
0272
0273 void sev_enable(struct boot_params *bp)
0274 {
0275     unsigned int eax, ebx, ecx, edx;
0276     struct msr m;
0277     bool snp;
0278
0279     /*
0280      * bp->cc_blob_address should only be set by boot/compressed kernel.
0281      * Initialize it to 0 to ensure that uninitialized values from
0282      * buggy bootloaders aren't propagated.
0283      */
0284     if (bp)
0285         bp->cc_blob_address = 0;
0286
0287     /*
0288      * Setup/preliminary detection of SNP. This will be sanity-checked
0289      * against CPUID/MSR values later.
0290      */
0291     snp = snp_init(bp);
0292
0293     /* Check for the SME/SEV support leaf */
0294     eax = 0x80000000;
0295     ecx = 0;
0296     native_cpuid(&eax, &ebx, &ecx, &edx);
0297     if (eax < 0x8000001f)
0298         return;
0299
0300     /*
0301      * Check for the SME/SEV feature:
0302      *   CPUID Fn8000_001F[EAX]
0303      *   - Bit 0 - Secure Memory Encryption support
0304      *   - Bit 1 - Secure Encrypted Virtualization support
0305      *   CPUID Fn8000_001F[EBX]
0306      *   - Bits 5:0 - Pagetable bit position used to indicate encryption
0307      */
0308     eax = 0x8000001f;
0309     ecx = 0;
0310     native_cpuid(&eax, &ebx, &ecx, &edx);
0311     /* Check whether SEV is supported */
0312     if (!(eax & BIT(1))) {
0313         if (snp)
0314             error("SEV-SNP support indicated by CC blob, but not CPUID.");
0315         return;
0316     }
0317
0318     /* Set the SME mask if this is an SEV guest. */
0319     boot_rdmsr(MSR_AMD64_SEV, &m);
0320     sev_status = m.q;
0321     if (!(sev_status & MSR_AMD64_SEV_ENABLED))
0322         return;
0323
0324     /* Negotiate the GHCB protocol version. */
0325     if (sev_status & MSR_AMD64_SEV_ES_ENABLED) {
0326         if (!sev_es_negotiate_protocol())
0327             sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_PROT_UNSUPPORTED);
0328     }
0329
0330     /*
0331      * SNP is supported in v2 of the GHCB spec which mandates support for HV
0332      * features.
0333      */
0334     if (sev_status & MSR_AMD64_SEV_SNP_ENABLED) {
0335         if (!(get_hv_features() & GHCB_HV_FT_SNP))
0336             sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
0337
0338         enforce_vmpl0();
0339     }
0340
0341     if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
0342         error("SEV-SNP supported indicated by CC blob, but not SEV status MSR.");
0343
0344     sme_me_mask = BIT_ULL(ebx & 0x3f);
0345 }
0346
0347 /* Search for Confidential Computing blob in the EFI config table. */
0348 static struct cc_blob_sev_info *find_cc_blob_efi(struct boot_params *bp)
0349 {
0350     unsigned long cfg_table_pa;
0351     unsigned int cfg_table_len;
0352     int ret;
0353
0354     ret = efi_get_conf_table(bp, &cfg_table_pa, &cfg_table_len);
0355     if (ret)
0356         return NULL;
0357
0358     return (struct cc_blob_sev_info *)efi_find_vendor_table(bp, cfg_table_pa,
0359                                 cfg_table_len,
0360                                 EFI_CC_BLOB_GUID);
0361 }
0362
0363 /*
0364  * Initial set up of SNP relies on information provided by the
0365  * Confidential Computing blob, which can be passed to the boot kernel
0366  * by firmware/bootloader in the following ways:
0367  *
0368  * - via an entry in the EFI config table
0369  * - via a setup_data structure, as defined by the Linux Boot Protocol
0370  *
0371  * Scan for the blob in that order.
0372  */
0373 static struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp)
0374 {
0375     struct cc_blob_sev_info *cc_info;
0376
0377     cc_info = find_cc_blob_efi(bp);
0378     if (cc_info)
0379         goto found_cc_info;
0380
0381     cc_info = find_cc_blob_setup_data(bp);
0382     if (!cc_info)
0383         return NULL;
0384
0385 found_cc_info:
0386     if (cc_info->magic != CC_BLOB_SEV_HDR_MAGIC)
0387         sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED);
0388
0389     return cc_info;
0390 }
0391
0392 /*
0393  * Indicate SNP based on presence of SNP-specific CC blob. Subsequent checks
0394  * will verify the SNP CPUID/MSR bits.
0395  */
0396 bool snp_init(struct boot_params *bp)
0397 {
0398     struct cc_blob_sev_info *cc_info;
0399
0400     if (!bp)
0401         return false;
0402
0403     cc_info = find_cc_blob(bp);
0404     if (!cc_info)
0405         return false;
0406
0407     /*
0408      * If a SNP-specific Confidential Computing blob is present, then
0409      * firmware/bootloader have indicated SNP support. Verifying this
0410      * involves CPUID checks which will be more reliable if the SNP
0411      * CPUID table is used. See comments over snp_setup_cpuid_table() for
0412      * more details.
0413      */
0414     setup_cpuid_table(cc_info);
0415
0416     /*
0417      * Pass run-time kernel a pointer to CC info via boot_params so EFI
0418      * config table doesn't need to be searched again during early startup
0419      * phase.
0420      */
0421     bp->cc_blob_address = (u32)(unsigned long)cc_info;
0422
0423     return true;
0424 }
0425
0426 void sev_prep_identity_maps(unsigned long top_level_pgt)
0427 {
0428     /*
0429      * The Confidential Computing blob is used very early in uncompressed
0430      * kernel to find the in-memory CPUID table to handle CPUID
0431      * instructions. Make sure an identity-mapping exists so it can be
0432      * accessed after switchover.
0433      */
0434     if (sev_snp_enabled()) {
0435         unsigned long cc_info_pa = boot_params->cc_blob_address;
0436         struct cc_blob_sev_info *cc_info;
0437
0438         kernel_add_identity_map(cc_info_pa, cc_info_pa + sizeof(*cc_info));
0439
0440         cc_info = (struct cc_blob_sev_info *)cc_info_pa;
0441         kernel_add_identity_map(cc_info->cpuid_phys, cc_info->cpuid_phys + cc_info->cpuid_len);
0442     }
0443
0444     sev_verify_cbit(top_level_pgt);
0445 }