kvm/x86_64/amx_test.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * amx tests
0004  *
0005  * Copyright (C) 2021, Intel, Inc.
0006  *
0007  * Tests for amx #NM exception and save/restore.
0008  */
0009
0010 #define _GNU_SOURCE /* for program_invocation_short_name */
0011 #include <fcntl.h>
0012 #include <stdio.h>
0013 #include <stdlib.h>
0014 #include <string.h>
0015 #include <sys/ioctl.h>
0016 #include <sys/syscall.h>
0017
0018 #include "test_util.h"
0019
0020 #include "kvm_util.h"
0021 #include "processor.h"
0022 #include "vmx.h"
0023
0024 #ifndef __x86_64__
0025 # error This test is 64-bit only
0026 #endif
0027
0028 #define NUM_TILES           8
0029 #define TILE_SIZE           1024
0030 #define XSAVE_SIZE          ((NUM_TILES * TILE_SIZE) + PAGE_SIZE)
0031
0032 /* Tile configuration associated: */
0033 #define MAX_TILES           16
0034 #define RESERVED_BYTES          14
0035
0036 #define XFEATURE_XTILECFG       17
0037 #define XFEATURE_XTILEDATA      18
0038 #define XFEATURE_MASK_XTILECFG      (1 << XFEATURE_XTILECFG)
0039 #define XFEATURE_MASK_XTILEDATA     (1 << XFEATURE_XTILEDATA)
0040 #define XFEATURE_MASK_XTILE     (XFEATURE_MASK_XTILECFG | XFEATURE_MASK_XTILEDATA)
0041
0042 #define TILE_CPUID          0x1d
0043 #define XSTATE_CPUID            0xd
0044 #define TILE_PALETTE_CPUID_SUBLEAVE 0x1
0045 #define XSTATE_USER_STATE_SUBLEAVE  0x0
0046
0047 #define XSAVE_HDR_OFFSET        512
0048
0049 struct xsave_data {
0050     u8 area[XSAVE_SIZE];
0051 } __aligned(64);
0052
0053 struct tile_config {
0054     u8  palette_id;
0055     u8  start_row;
0056     u8  reserved[RESERVED_BYTES];
0057     u16 colsb[MAX_TILES];
0058     u8  rows[MAX_TILES];
0059 };
0060
0061 struct tile_data {
0062     u8 data[NUM_TILES * TILE_SIZE];
0063 };
0064
0065 struct xtile_info {
0066     u16 bytes_per_tile;
0067     u16 bytes_per_row;
0068     u16 max_names;
0069     u16 max_rows;
0070     u32 xsave_offset;
0071     u32 xsave_size;
0072 };
0073
0074 static struct xtile_info xtile;
0075
0076 static inline u64 __xgetbv(u32 index)
0077 {
0078     u32 eax, edx;
0079
0080     asm volatile("xgetbv;"
0081              : "=a" (eax), "=d" (edx)
0082              : "c" (index));
0083     return eax + ((u64)edx << 32);
0084 }
0085
0086 static inline void __xsetbv(u32 index, u64 value)
0087 {
0088     u32 eax = value;
0089     u32 edx = value >> 32;
0090
0091     asm volatile("xsetbv" :: "a" (eax), "d" (edx), "c" (index));
0092 }
0093
0094 static inline void __ldtilecfg(void *cfg)
0095 {
0096     asm volatile(".byte 0xc4,0xe2,0x78,0x49,0x00"
0097              : : "a"(cfg));
0098 }
0099
0100 static inline void __tileloadd(void *tile)
0101 {
0102     asm volatile(".byte 0xc4,0xe2,0x7b,0x4b,0x04,0x10"
0103              : : "a"(tile), "d"(0));
0104 }
0105
0106 static inline void __tilerelease(void)
0107 {
0108     asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0" ::);
0109 }
0110
0111 static inline void __xsavec(struct xsave_data *data, uint64_t rfbm)
0112 {
0113     uint32_t rfbm_lo = rfbm;
0114     uint32_t rfbm_hi = rfbm >> 32;
0115
0116     asm volatile("xsavec (%%rdi)"
0117              : : "D" (data), "a" (rfbm_lo), "d" (rfbm_hi)
0118              : "memory");
0119 }
0120
0121 static inline void check_cpuid_xsave(void)
0122 {
0123     GUEST_ASSERT(this_cpu_has(X86_FEATURE_XSAVE));
0124     GUEST_ASSERT(this_cpu_has(X86_FEATURE_OSXSAVE));
0125 }
0126
0127 static bool check_xsave_supports_xtile(void)
0128 {
0129     return __xgetbv(0) & XFEATURE_MASK_XTILE;
0130 }
0131
0132 static bool enum_xtile_config(void)
0133 {
0134     u32 eax, ebx, ecx, edx;
0135
0136     __cpuid(TILE_CPUID, TILE_PALETTE_CPUID_SUBLEAVE, &eax, &ebx, &ecx, &edx);
0137     if (!eax || !ebx || !ecx)
0138         return false;
0139
0140     xtile.max_names = ebx >> 16;
0141     if (xtile.max_names < NUM_TILES)
0142         return false;
0143
0144     xtile.bytes_per_tile = eax >> 16;
0145     if (xtile.bytes_per_tile < TILE_SIZE)
0146         return false;
0147
0148     xtile.bytes_per_row = ebx;
0149     xtile.max_rows = ecx;
0150
0151     return true;
0152 }
0153
0154 static bool enum_xsave_tile(void)
0155 {
0156     u32 eax, ebx, ecx, edx;
0157
0158     __cpuid(XSTATE_CPUID, XFEATURE_XTILEDATA, &eax, &ebx, &ecx, &edx);
0159     if (!eax || !ebx)
0160         return false;
0161
0162     xtile.xsave_offset = ebx;
0163     xtile.xsave_size = eax;
0164
0165     return true;
0166 }
0167
0168 static bool check_xsave_size(void)
0169 {
0170     u32 eax, ebx, ecx, edx;
0171     bool valid = false;
0172
0173     __cpuid(XSTATE_CPUID, XSTATE_USER_STATE_SUBLEAVE, &eax, &ebx, &ecx, &edx);
0174     if (ebx && ebx <= XSAVE_SIZE)
0175         valid = true;
0176
0177     return valid;
0178 }
0179
0180 static bool check_xtile_info(void)
0181 {
0182     bool ret = false;
0183
0184     if (!check_xsave_size())
0185         return ret;
0186
0187     if (!enum_xsave_tile())
0188         return ret;
0189
0190     if (!enum_xtile_config())
0191         return ret;
0192
0193     if (sizeof(struct tile_data) >= xtile.xsave_size)
0194         ret = true;
0195
0196     return ret;
0197 }
0198
0199 static void set_tilecfg(struct tile_config *cfg)
0200 {
0201     int i;
0202
0203     /* Only palette id 1 */
0204     cfg->palette_id = 1;
0205     for (i = 0; i < xtile.max_names; i++) {
0206         cfg->colsb[i] = xtile.bytes_per_row;
0207         cfg->rows[i] = xtile.max_rows;
0208     }
0209 }
0210
0211 static void set_xstatebv(void *data, uint64_t bv)
0212 {
0213     *(uint64_t *)(data + XSAVE_HDR_OFFSET) = bv;
0214 }
0215
0216 static u64 get_xstatebv(void *data)
0217 {
0218     return *(u64 *)(data + XSAVE_HDR_OFFSET);
0219 }
0220
0221 static void init_regs(void)
0222 {
0223     uint64_t cr4, xcr0;
0224
0225     /* turn on CR4.OSXSAVE */
0226     cr4 = get_cr4();
0227     cr4 |= X86_CR4_OSXSAVE;
0228     set_cr4(cr4);
0229
0230     xcr0 = __xgetbv(0);
0231     xcr0 |= XFEATURE_MASK_XTILE;
0232     __xsetbv(0x0, xcr0);
0233 }
0234
0235 static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg,
0236                             struct tile_data *tiledata,
0237                             struct xsave_data *xsave_data)
0238 {
0239     init_regs();
0240     check_cpuid_xsave();
0241     GUEST_ASSERT(check_xsave_supports_xtile());
0242     GUEST_ASSERT(check_xtile_info());
0243
0244     /* check xtile configs */
0245     GUEST_ASSERT(xtile.xsave_offset == 2816);
0246     GUEST_ASSERT(xtile.xsave_size == 8192);
0247     GUEST_ASSERT(xtile.max_names == 8);
0248     GUEST_ASSERT(xtile.bytes_per_tile == 1024);
0249     GUEST_ASSERT(xtile.bytes_per_row == 64);
0250     GUEST_ASSERT(xtile.max_rows == 16);
0251     GUEST_SYNC(1);
0252
0253     /* xfd=0, enable amx */
0254     wrmsr(MSR_IA32_XFD, 0);
0255     GUEST_SYNC(2);
0256     GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0);
0257     set_tilecfg(amx_cfg);
0258     __ldtilecfg(amx_cfg);
0259     GUEST_SYNC(3);
0260     /* Check save/restore when trap to userspace */
0261     __tileloadd(tiledata);
0262     GUEST_SYNC(4);
0263     __tilerelease();
0264     GUEST_SYNC(5);
0265     /* bit 18 not in the XCOMP_BV after xsavec() */
0266     set_xstatebv(xsave_data, XFEATURE_MASK_XTILEDATA);
0267     __xsavec(xsave_data, XFEATURE_MASK_XTILEDATA);
0268     GUEST_ASSERT((get_xstatebv(xsave_data) & XFEATURE_MASK_XTILEDATA) == 0);
0269
0270     /* xfd=0x40000, disable amx tiledata */
0271     wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILEDATA);
0272     GUEST_SYNC(6);
0273     GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILEDATA);
0274     set_tilecfg(amx_cfg);
0275     __ldtilecfg(amx_cfg);
0276     /* Trigger #NM exception */
0277     __tileloadd(tiledata);
0278     GUEST_SYNC(10);
0279
0280     GUEST_DONE();
0281 }
0282
0283 void guest_nm_handler(struct ex_regs *regs)
0284 {
0285     /* Check if #NM is triggered by XFEATURE_MASK_XTILEDATA */
0286     GUEST_SYNC(7);
0287     GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA);
0288     GUEST_SYNC(8);
0289     GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILEDATA);
0290     /* Clear xfd_err */
0291     wrmsr(MSR_IA32_XFD_ERR, 0);
0292     /* xfd=0, enable amx */
0293     wrmsr(MSR_IA32_XFD, 0);
0294     GUEST_SYNC(9);
0295 }
0296
0297 int main(int argc, char *argv[])
0298 {
0299     struct kvm_regs regs1, regs2;
0300     struct kvm_vcpu *vcpu;
0301     struct kvm_vm *vm;
0302     struct kvm_run *run;
0303     struct kvm_x86_state *state;
0304     int xsave_restore_size;
0305     vm_vaddr_t amx_cfg, tiledata, xsavedata;
0306     struct ucall uc;
0307     u32 amx_offset;
0308     int stage, ret;
0309
0310     vm_xsave_require_permission(XSTATE_XTILE_DATA_BIT);
0311
0312     /* Create VM */
0313     vm = vm_create_with_one_vcpu(&vcpu, guest_code);
0314
0315     TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XSAVE));
0316     TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_AMX_TILE));
0317     TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILECFG));
0318     TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_XTILEDATA));
0319
0320     /* Get xsave/restore max size */
0321     xsave_restore_size = kvm_get_supported_cpuid_entry(0xd)->ecx;
0322
0323     run = vcpu->run;
0324     vcpu_regs_get(vcpu, &regs1);
0325
0326     /* Register #NM handler */
0327     vm_init_descriptor_tables(vm);
0328     vcpu_init_descriptor_tables(vcpu);
0329     vm_install_exception_handler(vm, NM_VECTOR, guest_nm_handler);
0330
0331     /* amx cfg for guest_code */
0332     amx_cfg = vm_vaddr_alloc_page(vm);
0333     memset(addr_gva2hva(vm, amx_cfg), 0x0, getpagesize());
0334
0335     /* amx tiledata for guest_code */
0336     tiledata = vm_vaddr_alloc_pages(vm, 2);
0337     memset(addr_gva2hva(vm, tiledata), rand() | 1, 2 * getpagesize());
0338
0339     /* xsave data for guest_code */
0340     xsavedata = vm_vaddr_alloc_pages(vm, 3);
0341     memset(addr_gva2hva(vm, xsavedata), 0, 3 * getpagesize());
0342     vcpu_args_set(vcpu, 3, amx_cfg, tiledata, xsavedata);
0343
0344     for (stage = 1; ; stage++) {
0345         vcpu_run(vcpu);
0346         TEST_ASSERT(run->exit_reason == KVM_EXIT_IO,
0347                 "Stage %d: unexpected exit reason: %u (%s),\n",
0348                 stage, run->exit_reason,
0349                 exit_reason_str(run->exit_reason));
0350
0351         switch (get_ucall(vcpu, &uc)) {
0352         case UCALL_ABORT:
0353             REPORT_GUEST_ASSERT(uc);
0354             /* NOT REACHED */
0355         case UCALL_SYNC:
0356             switch (uc.args[1]) {
0357             case 1:
0358             case 2:
0359             case 3:
0360             case 5:
0361             case 6:
0362             case 7:
0363             case 8:
0364                 fprintf(stderr, "GUEST_SYNC(%ld)\n", uc.args[1]);
0365                 break;
0366             case 4:
0367             case 10:
0368                 fprintf(stderr,
0369                 "GUEST_SYNC(%ld), check save/restore status\n", uc.args[1]);
0370
0371                 /* Compacted mode, get amx offset by xsave area
0372                  * size subtract 8K amx size.
0373                  */
0374                 amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE;
0375                 state = vcpu_save_state(vcpu);
0376                 void *amx_start = (void *)state->xsave + amx_offset;
0377                 void *tiles_data = (void *)addr_gva2hva(vm, tiledata);
0378                 /* Only check TMM0 register, 1 tile */
0379                 ret = memcmp(amx_start, tiles_data, TILE_SIZE);
0380                 TEST_ASSERT(ret == 0, "memcmp failed, ret=%d\n", ret);
0381                 kvm_x86_state_cleanup(state);
0382                 break;
0383             case 9:
0384                 fprintf(stderr,
0385                 "GUEST_SYNC(%ld), #NM exception and enable amx\n", uc.args[1]);
0386                 break;
0387             }
0388             break;
0389         case UCALL_DONE:
0390             fprintf(stderr, "UCALL_DONE\n");
0391             goto done;
0392         default:
0393             TEST_FAIL("Unknown ucall %lu", uc.cmd);
0394         }
0395
0396         state = vcpu_save_state(vcpu);
0397         memset(&regs1, 0, sizeof(regs1));
0398         vcpu_regs_get(vcpu, &regs1);
0399
0400         kvm_vm_release(vm);
0401
0402         /* Restore state in a new VM.  */
0403         vcpu = vm_recreate_with_one_vcpu(vm);
0404         vcpu_load_state(vcpu, state);
0405         run = vcpu->run;
0406         kvm_x86_state_cleanup(state);
0407
0408         memset(&regs2, 0, sizeof(regs2));
0409         vcpu_regs_get(vcpu, &regs2);
0410         TEST_ASSERT(!memcmp(&regs1, &regs2, sizeof(regs2)),
0411                 "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
0412                 (ulong) regs2.rdi, (ulong) regs2.rsi);
0413     }
0414 done:
0415     kvm_vm_free(vm);
0416 }