0001
0002
0003
0004
0005
0006
0007
0008
0009
0010 #define DISABLE_BRANCH_PROFILING
0011
0012
0013
0014
0015
0016
0017 #define __pa(x) ((unsigned long)(x))
0018 #define __va(x) ((void *)((unsigned long)(x)))
0019
0020
0021
0022
0023
0024
0025
0026 #undef CONFIG_PARAVIRT
0027 #undef CONFIG_PARAVIRT_XXL
0028 #undef CONFIG_PARAVIRT_SPINLOCKS
0029
0030
0031
0032
0033
0034
0035
0036
0037 #define USE_EARLY_PGTABLE_L5
0038
0039 #include <linux/kernel.h>
0040 #include <linux/mm.h>
0041 #include <linux/mem_encrypt.h>
0042 #include <linux/cc_platform.h>
0043
0044 #include <asm/setup.h>
0045 #include <asm/sections.h>
0046 #include <asm/cmdline.h>
0047 #include <asm/coco.h>
0048 #include <asm/sev.h>
0049
0050 #include "mm_internal.h"
0051
0052 #define PGD_FLAGS _KERNPG_TABLE_NOENC
0053 #define P4D_FLAGS _KERNPG_TABLE_NOENC
0054 #define PUD_FLAGS _KERNPG_TABLE_NOENC
0055 #define PMD_FLAGS _KERNPG_TABLE_NOENC
0056
0057 #define PMD_FLAGS_LARGE (__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL)
0058
0059 #define PMD_FLAGS_DEC PMD_FLAGS_LARGE
0060 #define PMD_FLAGS_DEC_WP ((PMD_FLAGS_DEC & ~_PAGE_LARGE_CACHE_MASK) | \
0061 (_PAGE_PAT_LARGE | _PAGE_PWT))
0062
0063 #define PMD_FLAGS_ENC (PMD_FLAGS_LARGE | _PAGE_ENC)
0064
0065 #define PTE_FLAGS (__PAGE_KERNEL_EXEC & ~_PAGE_GLOBAL)
0066
0067 #define PTE_FLAGS_DEC PTE_FLAGS
0068 #define PTE_FLAGS_DEC_WP ((PTE_FLAGS_DEC & ~_PAGE_CACHE_MASK) | \
0069 (_PAGE_PAT | _PAGE_PWT))
0070
0071 #define PTE_FLAGS_ENC (PTE_FLAGS | _PAGE_ENC)
0072
0073 struct sme_populate_pgd_data {
0074 void *pgtable_area;
0075 pgd_t *pgd;
0076
0077 pmdval_t pmd_flags;
0078 pteval_t pte_flags;
0079 unsigned long paddr;
0080
0081 unsigned long vaddr;
0082 unsigned long vaddr_end;
0083 };
0084
0085
0086
0087
0088
0089
0090
0091
0092
0093
0094
0095
0096 static char sme_workarea[2 * PMD_PAGE_SIZE] __section(".init.scratch");
0097
0098 static char sme_cmdline_arg[] __initdata = "mem_encrypt";
0099 static char sme_cmdline_on[] __initdata = "on";
0100 static char sme_cmdline_off[] __initdata = "off";
0101
0102 static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
0103 {
0104 unsigned long pgd_start, pgd_end, pgd_size;
0105 pgd_t *pgd_p;
0106
0107 pgd_start = ppd->vaddr & PGDIR_MASK;
0108 pgd_end = ppd->vaddr_end & PGDIR_MASK;
0109
0110 pgd_size = (((pgd_end - pgd_start) / PGDIR_SIZE) + 1) * sizeof(pgd_t);
0111
0112 pgd_p = ppd->pgd + pgd_index(ppd->vaddr);
0113
0114 memset(pgd_p, 0, pgd_size);
0115 }
0116
0117 static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
0118 {
0119 pgd_t *pgd;
0120 p4d_t *p4d;
0121 pud_t *pud;
0122 pmd_t *pmd;
0123
0124 pgd = ppd->pgd + pgd_index(ppd->vaddr);
0125 if (pgd_none(*pgd)) {
0126 p4d = ppd->pgtable_area;
0127 memset(p4d, 0, sizeof(*p4d) * PTRS_PER_P4D);
0128 ppd->pgtable_area += sizeof(*p4d) * PTRS_PER_P4D;
0129 set_pgd(pgd, __pgd(PGD_FLAGS | __pa(p4d)));
0130 }
0131
0132 p4d = p4d_offset(pgd, ppd->vaddr);
0133 if (p4d_none(*p4d)) {
0134 pud = ppd->pgtable_area;
0135 memset(pud, 0, sizeof(*pud) * PTRS_PER_PUD);
0136 ppd->pgtable_area += sizeof(*pud) * PTRS_PER_PUD;
0137 set_p4d(p4d, __p4d(P4D_FLAGS | __pa(pud)));
0138 }
0139
0140 pud = pud_offset(p4d, ppd->vaddr);
0141 if (pud_none(*pud)) {
0142 pmd = ppd->pgtable_area;
0143 memset(pmd, 0, sizeof(*pmd) * PTRS_PER_PMD);
0144 ppd->pgtable_area += sizeof(*pmd) * PTRS_PER_PMD;
0145 set_pud(pud, __pud(PUD_FLAGS | __pa(pmd)));
0146 }
0147
0148 if (pud_large(*pud))
0149 return NULL;
0150
0151 return pud;
0152 }
0153
0154 static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
0155 {
0156 pud_t *pud;
0157 pmd_t *pmd;
0158
0159 pud = sme_prepare_pgd(ppd);
0160 if (!pud)
0161 return;
0162
0163 pmd = pmd_offset(pud, ppd->vaddr);
0164 if (pmd_large(*pmd))
0165 return;
0166
0167 set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags));
0168 }
0169
0170 static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
0171 {
0172 pud_t *pud;
0173 pmd_t *pmd;
0174 pte_t *pte;
0175
0176 pud = sme_prepare_pgd(ppd);
0177 if (!pud)
0178 return;
0179
0180 pmd = pmd_offset(pud, ppd->vaddr);
0181 if (pmd_none(*pmd)) {
0182 pte = ppd->pgtable_area;
0183 memset(pte, 0, sizeof(*pte) * PTRS_PER_PTE);
0184 ppd->pgtable_area += sizeof(*pte) * PTRS_PER_PTE;
0185 set_pmd(pmd, __pmd(PMD_FLAGS | __pa(pte)));
0186 }
0187
0188 if (pmd_large(*pmd))
0189 return;
0190
0191 pte = pte_offset_map(pmd, ppd->vaddr);
0192 if (pte_none(*pte))
0193 set_pte(pte, __pte(ppd->paddr | ppd->pte_flags));
0194 }
0195
0196 static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
0197 {
0198 while (ppd->vaddr < ppd->vaddr_end) {
0199 sme_populate_pgd_large(ppd);
0200
0201 ppd->vaddr += PMD_PAGE_SIZE;
0202 ppd->paddr += PMD_PAGE_SIZE;
0203 }
0204 }
0205
0206 static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
0207 {
0208 while (ppd->vaddr < ppd->vaddr_end) {
0209 sme_populate_pgd(ppd);
0210
0211 ppd->vaddr += PAGE_SIZE;
0212 ppd->paddr += PAGE_SIZE;
0213 }
0214 }
0215
0216 static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
0217 pmdval_t pmd_flags, pteval_t pte_flags)
0218 {
0219 unsigned long vaddr_end;
0220
0221 ppd->pmd_flags = pmd_flags;
0222 ppd->pte_flags = pte_flags;
0223
0224
0225 vaddr_end = ppd->vaddr_end;
0226
0227
0228 ppd->vaddr_end = ALIGN(ppd->vaddr, PMD_PAGE_SIZE);
0229 __sme_map_range_pte(ppd);
0230
0231
0232 ppd->vaddr_end = vaddr_end & PMD_PAGE_MASK;
0233 __sme_map_range_pmd(ppd);
0234
0235
0236 ppd->vaddr_end = vaddr_end;
0237 __sme_map_range_pte(ppd);
0238 }
0239
0240 static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
0241 {
0242 __sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC);
0243 }
0244
0245 static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
0246 {
0247 __sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC);
0248 }
0249
0250 static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
0251 {
0252 __sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP);
0253 }
0254
0255 static unsigned long __init sme_pgtable_calc(unsigned long len)
0256 {
0257 unsigned long entries = 0, tables = 0;
0258
0259
0260
0261
0262
0263
0264
0265
0266
0267
0268
0269
0270
0271
0272
0273 if (PTRS_PER_P4D > 1)
0274 entries += (DIV_ROUND_UP(len, PGDIR_SIZE) + 1) * sizeof(p4d_t) * PTRS_PER_P4D;
0275 entries += (DIV_ROUND_UP(len, P4D_SIZE) + 1) * sizeof(pud_t) * PTRS_PER_PUD;
0276 entries += (DIV_ROUND_UP(len, PUD_SIZE) + 1) * sizeof(pmd_t) * PTRS_PER_PMD;
0277 entries += 2 * sizeof(pte_t) * PTRS_PER_PTE;
0278
0279
0280
0281
0282
0283
0284 if (PTRS_PER_P4D > 1)
0285 tables += DIV_ROUND_UP(entries, PGDIR_SIZE) * sizeof(p4d_t) * PTRS_PER_P4D;
0286 tables += DIV_ROUND_UP(entries, P4D_SIZE) * sizeof(pud_t) * PTRS_PER_PUD;
0287 tables += DIV_ROUND_UP(entries, PUD_SIZE) * sizeof(pmd_t) * PTRS_PER_PMD;
0288
0289 return entries + tables;
0290 }
0291
0292 void __init sme_encrypt_kernel(struct boot_params *bp)
0293 {
0294 unsigned long workarea_start, workarea_end, workarea_len;
0295 unsigned long execute_start, execute_end, execute_len;
0296 unsigned long kernel_start, kernel_end, kernel_len;
0297 unsigned long initrd_start, initrd_end, initrd_len;
0298 struct sme_populate_pgd_data ppd;
0299 unsigned long pgtable_area_len;
0300 unsigned long decrypted_base;
0301
0302
0303
0304
0305
0306
0307
0308 if (!sme_get_me_mask() || sev_status & MSR_AMD64_SEV_ENABLED)
0309 return;
0310
0311
0312
0313
0314
0315
0316
0317
0318
0319
0320
0321
0322
0323
0324
0325
0326
0327 kernel_start = __pa_symbol(_text);
0328 kernel_end = ALIGN(__pa_symbol(_end), PMD_PAGE_SIZE);
0329 kernel_len = kernel_end - kernel_start;
0330
0331 initrd_start = 0;
0332 initrd_end = 0;
0333 initrd_len = 0;
0334 #ifdef CONFIG_BLK_DEV_INITRD
0335 initrd_len = (unsigned long)bp->hdr.ramdisk_size |
0336 ((unsigned long)bp->ext_ramdisk_size << 32);
0337 if (initrd_len) {
0338 initrd_start = (unsigned long)bp->hdr.ramdisk_image |
0339 ((unsigned long)bp->ext_ramdisk_image << 32);
0340 initrd_end = PAGE_ALIGN(initrd_start + initrd_len);
0341 initrd_len = initrd_end - initrd_start;
0342 }
0343 #endif
0344
0345
0346
0347
0348
0349 asm ("lea sme_workarea(%%rip), %0"
0350 : "=r" (workarea_start)
0351 : "p" (sme_workarea));
0352
0353
0354
0355
0356
0357
0358
0359
0360
0361
0362 execute_start = workarea_start;
0363 execute_end = execute_start + (PAGE_SIZE * 2) + PMD_PAGE_SIZE;
0364 execute_len = execute_end - execute_start;
0365
0366
0367
0368
0369
0370 pgtable_area_len = sizeof(pgd_t) * PTRS_PER_PGD;
0371 pgtable_area_len += sme_pgtable_calc(execute_end - kernel_start) * 2;
0372 if (initrd_len)
0373 pgtable_area_len += sme_pgtable_calc(initrd_len) * 2;
0374
0375
0376 pgtable_area_len += sme_pgtable_calc(execute_len + pgtable_area_len);
0377
0378
0379
0380
0381
0382
0383
0384
0385 workarea_len = execute_len + pgtable_area_len;
0386 workarea_end = ALIGN(workarea_start + workarea_len, PMD_PAGE_SIZE);
0387
0388
0389
0390
0391
0392
0393
0394
0395 ppd.pgtable_area = (void *)execute_end;
0396
0397
0398
0399
0400
0401 ppd.pgd = (pgd_t *)native_read_cr3_pa();
0402 ppd.paddr = workarea_start;
0403 ppd.vaddr = workarea_start;
0404 ppd.vaddr_end = workarea_end;
0405 sme_map_range_decrypted(&ppd);
0406
0407
0408 native_write_cr3(__native_read_cr3());
0409
0410
0411
0412
0413
0414
0415
0416 ppd.pgd = ppd.pgtable_area;
0417 memset(ppd.pgd, 0, sizeof(pgd_t) * PTRS_PER_PGD);
0418 ppd.pgtable_area += sizeof(pgd_t) * PTRS_PER_PGD;
0419
0420
0421
0422
0423
0424
0425
0426 decrypted_base = (pgd_index(workarea_end) + 1) & (PTRS_PER_PGD - 1);
0427 if (initrd_len) {
0428 unsigned long check_base;
0429
0430 check_base = (pgd_index(initrd_end) + 1) & (PTRS_PER_PGD - 1);
0431 decrypted_base = max(decrypted_base, check_base);
0432 }
0433 decrypted_base <<= PGDIR_SHIFT;
0434
0435
0436 ppd.paddr = kernel_start;
0437 ppd.vaddr = kernel_start;
0438 ppd.vaddr_end = kernel_end;
0439 sme_map_range_encrypted(&ppd);
0440
0441
0442 ppd.paddr = kernel_start;
0443 ppd.vaddr = kernel_start + decrypted_base;
0444 ppd.vaddr_end = kernel_end + decrypted_base;
0445 sme_map_range_decrypted_wp(&ppd);
0446
0447 if (initrd_len) {
0448
0449 ppd.paddr = initrd_start;
0450 ppd.vaddr = initrd_start;
0451 ppd.vaddr_end = initrd_end;
0452 sme_map_range_encrypted(&ppd);
0453
0454
0455
0456 ppd.paddr = initrd_start;
0457 ppd.vaddr = initrd_start + decrypted_base;
0458 ppd.vaddr_end = initrd_end + decrypted_base;
0459 sme_map_range_decrypted_wp(&ppd);
0460 }
0461
0462
0463 ppd.paddr = workarea_start;
0464 ppd.vaddr = workarea_start;
0465 ppd.vaddr_end = workarea_end;
0466 sme_map_range_decrypted(&ppd);
0467
0468 ppd.paddr = workarea_start;
0469 ppd.vaddr = workarea_start + decrypted_base;
0470 ppd.vaddr_end = workarea_end + decrypted_base;
0471 sme_map_range_decrypted(&ppd);
0472
0473
0474 sme_encrypt_execute(kernel_start, kernel_start + decrypted_base,
0475 kernel_len, workarea_start, (unsigned long)ppd.pgd);
0476
0477 if (initrd_len)
0478 sme_encrypt_execute(initrd_start, initrd_start + decrypted_base,
0479 initrd_len, workarea_start,
0480 (unsigned long)ppd.pgd);
0481
0482
0483
0484
0485
0486
0487 ppd.vaddr = kernel_start + decrypted_base;
0488 ppd.vaddr_end = kernel_end + decrypted_base;
0489 sme_clear_pgd(&ppd);
0490
0491 if (initrd_len) {
0492 ppd.vaddr = initrd_start + decrypted_base;
0493 ppd.vaddr_end = initrd_end + decrypted_base;
0494 sme_clear_pgd(&ppd);
0495 }
0496
0497 ppd.vaddr = workarea_start + decrypted_base;
0498 ppd.vaddr_end = workarea_end + decrypted_base;
0499 sme_clear_pgd(&ppd);
0500
0501
0502 native_write_cr3(__native_read_cr3());
0503 }
0504
0505 void __init sme_enable(struct boot_params *bp)
0506 {
0507 const char *cmdline_ptr, *cmdline_arg, *cmdline_on, *cmdline_off;
0508 unsigned int eax, ebx, ecx, edx;
0509 unsigned long feature_mask;
0510 bool active_by_default;
0511 unsigned long me_mask;
0512 char buffer[16];
0513 bool snp;
0514 u64 msr;
0515
0516 snp = snp_init(bp);
0517
0518
0519 eax = 0x80000000;
0520 ecx = 0;
0521 native_cpuid(&eax, &ebx, &ecx, &edx);
0522 if (eax < 0x8000001f)
0523 return;
0524
0525 #define AMD_SME_BIT BIT(0)
0526 #define AMD_SEV_BIT BIT(1)
0527
0528
0529
0530
0531
0532
0533
0534
0535
0536 eax = 0x8000001f;
0537 ecx = 0;
0538 native_cpuid(&eax, &ebx, &ecx, &edx);
0539
0540 if (!(eax & (AMD_SEV_BIT | AMD_SME_BIT)))
0541 return;
0542
0543 me_mask = 1UL << (ebx & 0x3f);
0544
0545
0546 sev_status = __rdmsr(MSR_AMD64_SEV);
0547 feature_mask = (sev_status & MSR_AMD64_SEV_ENABLED) ? AMD_SEV_BIT : AMD_SME_BIT;
0548
0549
0550 if (snp && !(sev_status & MSR_AMD64_SEV_SNP_ENABLED))
0551 snp_abort();
0552
0553
0554 if (feature_mask == AMD_SME_BIT) {
0555
0556
0557
0558
0559
0560
0561
0562
0563
0564 eax = 1;
0565 ecx = 0;
0566 native_cpuid(&eax, &ebx, &ecx, &edx);
0567 if (ecx & BIT(31))
0568 return;
0569
0570
0571 msr = __rdmsr(MSR_AMD64_SYSCFG);
0572 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
0573 return;
0574 } else {
0575
0576 sme_me_mask = me_mask;
0577 goto out;
0578 }
0579
0580
0581
0582
0583
0584
0585 asm ("lea sme_cmdline_arg(%%rip), %0"
0586 : "=r" (cmdline_arg)
0587 : "p" (sme_cmdline_arg));
0588 asm ("lea sme_cmdline_on(%%rip), %0"
0589 : "=r" (cmdline_on)
0590 : "p" (sme_cmdline_on));
0591 asm ("lea sme_cmdline_off(%%rip), %0"
0592 : "=r" (cmdline_off)
0593 : "p" (sme_cmdline_off));
0594
0595 if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT))
0596 active_by_default = true;
0597 else
0598 active_by_default = false;
0599
0600 cmdline_ptr = (const char *)((u64)bp->hdr.cmd_line_ptr |
0601 ((u64)bp->ext_cmd_line_ptr << 32));
0602
0603 cmdline_find_option(cmdline_ptr, cmdline_arg, buffer, sizeof(buffer));
0604
0605 if (!strncmp(buffer, cmdline_on, sizeof(buffer)))
0606 sme_me_mask = me_mask;
0607 else if (!strncmp(buffer, cmdline_off, sizeof(buffer)))
0608 sme_me_mask = 0;
0609 else
0610 sme_me_mask = active_by_default ? me_mask : 0;
0611 out:
0612 if (sme_me_mask) {
0613 physical_mask &= ~sme_me_mask;
0614 cc_set_vendor(CC_VENDOR_AMD);
0615 cc_set_mask(sme_me_mask);
0616 }
0617 }