selftests/kvm/max_guest_memory_test.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 #define _GNU_SOURCE
0003
0004 #include <stdio.h>
0005 #include <stdlib.h>
0006 #include <pthread.h>
0007 #include <semaphore.h>
0008 #include <sys/types.h>
0009 #include <signal.h>
0010 #include <errno.h>
0011 #include <linux/bitmap.h>
0012 #include <linux/bitops.h>
0013 #include <linux/atomic.h>
0014
0015 #include "kvm_util.h"
0016 #include "test_util.h"
0017 #include "guest_modes.h"
0018 #include "processor.h"
0019
0020 static void guest_code(uint64_t start_gpa, uint64_t end_gpa, uint64_t stride)
0021 {
0022     uint64_t gpa;
0023
0024     for (gpa = start_gpa; gpa < end_gpa; gpa += stride)
0025         *((volatile uint64_t *)gpa) = gpa;
0026
0027     GUEST_DONE();
0028 }
0029
0030 struct vcpu_info {
0031     struct kvm_vcpu *vcpu;
0032     uint64_t start_gpa;
0033     uint64_t end_gpa;
0034 };
0035
0036 static int nr_vcpus;
0037 static atomic_t rendezvous;
0038
0039 static void rendezvous_with_boss(void)
0040 {
0041     int orig = atomic_read(&rendezvous);
0042
0043     if (orig > 0) {
0044         atomic_dec_and_test(&rendezvous);
0045         while (atomic_read(&rendezvous) > 0)
0046             cpu_relax();
0047     } else {
0048         atomic_inc(&rendezvous);
0049         while (atomic_read(&rendezvous) < 0)
0050             cpu_relax();
0051     }
0052 }
0053
0054 static void run_vcpu(struct kvm_vcpu *vcpu)
0055 {
0056     vcpu_run(vcpu);
0057     ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_DONE);
0058 }
0059
0060 static void *vcpu_worker(void *data)
0061 {
0062     struct vcpu_info *info = data;
0063     struct kvm_vcpu *vcpu = info->vcpu;
0064     struct kvm_vm *vm = vcpu->vm;
0065     struct kvm_sregs sregs;
0066     struct kvm_regs regs;
0067
0068     vcpu_args_set(vcpu, 3, info->start_gpa, info->end_gpa, vm->page_size);
0069
0070     /* Snapshot regs before the first run. */
0071     vcpu_regs_get(vcpu, &regs);
0072     rendezvous_with_boss();
0073
0074     run_vcpu(vcpu);
0075     rendezvous_with_boss();
0076     vcpu_regs_set(vcpu, &regs);
0077     vcpu_sregs_get(vcpu, &sregs);
0078 #ifdef __x86_64__
0079     /* Toggle CR0.WP to trigger a MMU context reset. */
0080     sregs.cr0 ^= X86_CR0_WP;
0081 #endif
0082     vcpu_sregs_set(vcpu, &sregs);
0083     rendezvous_with_boss();
0084
0085     run_vcpu(vcpu);
0086     rendezvous_with_boss();
0087
0088     return NULL;
0089 }
0090
0091 static pthread_t *spawn_workers(struct kvm_vm *vm, struct kvm_vcpu **vcpus,
0092                 uint64_t start_gpa, uint64_t end_gpa)
0093 {
0094     struct vcpu_info *info;
0095     uint64_t gpa, nr_bytes;
0096     pthread_t *threads;
0097     int i;
0098
0099     threads = malloc(nr_vcpus * sizeof(*threads));
0100     TEST_ASSERT(threads, "Failed to allocate vCPU threads");
0101
0102     info = malloc(nr_vcpus * sizeof(*info));
0103     TEST_ASSERT(info, "Failed to allocate vCPU gpa ranges");
0104
0105     nr_bytes = ((end_gpa - start_gpa) / nr_vcpus) &
0106             ~((uint64_t)vm->page_size - 1);
0107     TEST_ASSERT(nr_bytes, "C'mon, no way you have %d CPUs", nr_vcpus);
0108
0109     for (i = 0, gpa = start_gpa; i < nr_vcpus; i++, gpa += nr_bytes) {
0110         info[i].vcpu = vcpus[i];
0111         info[i].start_gpa = gpa;
0112         info[i].end_gpa = gpa + nr_bytes;
0113         pthread_create(&threads[i], NULL, vcpu_worker, &info[i]);
0114     }
0115     return threads;
0116 }
0117
0118 static void rendezvous_with_vcpus(struct timespec *time, const char *name)
0119 {
0120     int i, rendezvoused;
0121
0122     pr_info("Waiting for vCPUs to finish %s...\n", name);
0123
0124     rendezvoused = atomic_read(&rendezvous);
0125     for (i = 0; abs(rendezvoused) != 1; i++) {
0126         usleep(100);
0127         if (!(i & 0x3f))
0128             pr_info("\r%d vCPUs haven't rendezvoused...",
0129                 abs(rendezvoused) - 1);
0130         rendezvoused = atomic_read(&rendezvous);
0131     }
0132
0133     clock_gettime(CLOCK_MONOTONIC, time);
0134
0135     /* Release the vCPUs after getting the time of the previous action. */
0136     pr_info("\rAll vCPUs finished %s, releasing...\n", name);
0137     if (rendezvoused > 0)
0138         atomic_set(&rendezvous, -nr_vcpus - 1);
0139     else
0140         atomic_set(&rendezvous, nr_vcpus + 1);
0141 }
0142
0143 static void calc_default_nr_vcpus(void)
0144 {
0145     cpu_set_t possible_mask;
0146     int r;
0147
0148     r = sched_getaffinity(0, sizeof(possible_mask), &possible_mask);
0149     TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)",
0150             errno, strerror(errno));
0151
0152     nr_vcpus = CPU_COUNT(&possible_mask) * 3/4;
0153     TEST_ASSERT(nr_vcpus > 0, "Uh, no CPUs?");
0154 }
0155
0156 int main(int argc, char *argv[])
0157 {
0158     /*
0159      * Skip the first 4gb and slot0.  slot0 maps <1gb and is used to back
0160      * the guest's code, stack, and page tables.  Because selftests creates
0161      * an IRQCHIP, a.k.a. a local APIC, KVM creates an internal memslot
0162      * just below the 4gb boundary.  This test could create memory at
0163      * 1gb-3gb,but it's simpler to skip straight to 4gb.
0164      */
0165     const uint64_t size_1gb = (1 << 30);
0166     const uint64_t start_gpa = (4ull * size_1gb);
0167     const int first_slot = 1;
0168
0169     struct timespec time_start, time_run1, time_reset, time_run2;
0170     uint64_t max_gpa, gpa, slot_size, max_mem, i;
0171     int max_slots, slot, opt, fd;
0172     bool hugepages = false;
0173     struct kvm_vcpu **vcpus;
0174     pthread_t *threads;
0175     struct kvm_vm *vm;
0176     void *mem;
0177
0178     /*
0179      * Default to 2gb so that maxing out systems with MAXPHADDR=46, which
0180      * are quite common for x86, requires changing only max_mem (KVM allows
0181      * 32k memslots, 32k * 2gb == ~64tb of guest memory).
0182      */
0183     slot_size = 2 * size_1gb;
0184
0185     max_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);
0186     TEST_ASSERT(max_slots > first_slot, "KVM is broken");
0187
0188     /* All KVM MMUs should be able to survive a 128gb guest. */
0189     max_mem = 128 * size_1gb;
0190
0191     calc_default_nr_vcpus();
0192
0193     while ((opt = getopt(argc, argv, "c:h:m:s:H")) != -1) {
0194         switch (opt) {
0195         case 'c':
0196             nr_vcpus = atoi(optarg);
0197             TEST_ASSERT(nr_vcpus > 0, "number of vcpus must be >0");
0198             break;
0199         case 'm':
0200             max_mem = atoi(optarg) * size_1gb;
0201             TEST_ASSERT(max_mem > 0, "memory size must be >0");
0202             break;
0203         case 's':
0204             slot_size = atoi(optarg) * size_1gb;
0205             TEST_ASSERT(slot_size > 0, "slot size must be >0");
0206             break;
0207         case 'H':
0208             hugepages = true;
0209             break;
0210         case 'h':
0211         default:
0212             printf("usage: %s [-c nr_vcpus] [-m max_mem_in_gb] [-s slot_size_in_gb] [-H]\n", argv[0]);
0213             exit(1);
0214         }
0215     }
0216
0217     vcpus = malloc(nr_vcpus * sizeof(*vcpus));
0218     TEST_ASSERT(vcpus, "Failed to allocate vCPU array");
0219
0220     vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus);
0221
0222     max_gpa = vm->max_gfn << vm->page_shift;
0223     TEST_ASSERT(max_gpa > (4 * slot_size), "MAXPHYADDR <4gb ");
0224
0225     fd = kvm_memfd_alloc(slot_size, hugepages);
0226     mem = mmap(NULL, slot_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
0227     TEST_ASSERT(mem != MAP_FAILED, "mmap() failed");
0228
0229     TEST_ASSERT(!madvise(mem, slot_size, MADV_NOHUGEPAGE), "madvise() failed");
0230
0231     /* Pre-fault the memory to avoid taking mmap_sem on guest page faults. */
0232     for (i = 0; i < slot_size; i += vm->page_size)
0233         ((uint8_t *)mem)[i] = 0xaa;
0234
0235     gpa = 0;
0236     for (slot = first_slot; slot < max_slots; slot++) {
0237         gpa = start_gpa + ((slot - first_slot) * slot_size);
0238         if (gpa + slot_size > max_gpa)
0239             break;
0240
0241         if ((gpa - start_gpa) >= max_mem)
0242             break;
0243
0244         vm_set_user_memory_region(vm, slot, 0, gpa, slot_size, mem);
0245
0246 #ifdef __x86_64__
0247         /* Identity map memory in the guest using 1gb pages. */
0248         for (i = 0; i < slot_size; i += size_1gb)
0249             __virt_pg_map(vm, gpa + i, gpa + i, PG_LEVEL_1G);
0250 #else
0251         for (i = 0; i < slot_size; i += vm->page_size)
0252             virt_pg_map(vm, gpa + i, gpa + i);
0253 #endif
0254     }
0255
0256     atomic_set(&rendezvous, nr_vcpus + 1);
0257     threads = spawn_workers(vm, vcpus, start_gpa, gpa);
0258
0259     free(vcpus);
0260     vcpus = NULL;
0261
0262     pr_info("Running with %lugb of guest memory and %u vCPUs\n",
0263         (gpa - start_gpa) / size_1gb, nr_vcpus);
0264
0265     rendezvous_with_vcpus(&time_start, "spawning");
0266     rendezvous_with_vcpus(&time_run1, "run 1");
0267     rendezvous_with_vcpus(&time_reset, "reset");
0268     rendezvous_with_vcpus(&time_run2, "run 2");
0269
0270     time_run2  = timespec_sub(time_run2,   time_reset);
0271     time_reset = timespec_sub(time_reset, time_run1);
0272     time_run1  = timespec_sub(time_run1,   time_start);
0273
0274     pr_info("run1 = %ld.%.9lds, reset = %ld.%.9lds, run2 =  %ld.%.9lds\n",
0275         time_run1.tv_sec, time_run1.tv_nsec,
0276         time_reset.tv_sec, time_reset.tv_nsec,
0277         time_run2.tv_sec, time_run2.tv_nsec);
0278
0279     /*
0280      * Delete even numbered slots (arbitrary) and unmap the first half of
0281      * the backing (also arbitrary) to verify KVM correctly drops all
0282      * references to the removed regions.
0283      */
0284     for (slot = (slot - 1) & ~1ull; slot >= first_slot; slot -= 2)
0285         vm_set_user_memory_region(vm, slot, 0, 0, 0, NULL);
0286
0287     munmap(mem, slot_size / 2);
0288
0289     /* Sanity check that the vCPUs actually ran. */
0290     for (i = 0; i < nr_vcpus; i++)
0291         pthread_join(threads[i], NULL);
0292
0293     /*
0294      * Deliberately exit without deleting the remaining memslots or closing
0295      * kvm_fd to test cleanup via mmu_notifier.release.
0296      */
0297 }