selftests/vm/mremap_test.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * Copyright 2020 Google LLC
0004  */
0005 #define _GNU_SOURCE
0006
0007 #include <errno.h>
0008 #include <stdlib.h>
0009 #include <stdio.h>
0010 #include <string.h>
0011 #include <sys/mman.h>
0012 #include <time.h>
0013 #include <stdbool.h>
0014
0015 #include "../kselftest.h"
0016
0017 #define EXPECT_SUCCESS 0
0018 #define EXPECT_FAILURE 1
0019 #define NON_OVERLAPPING 0
0020 #define OVERLAPPING 1
0021 #define NS_PER_SEC 1000000000ULL
0022 #define VALIDATION_DEFAULT_THRESHOLD 4  /* 4MB */
0023 #define VALIDATION_NO_THRESHOLD 0   /* Verify the entire region */
0024
0025 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
0026
0027 struct config {
0028     unsigned long long src_alignment;
0029     unsigned long long dest_alignment;
0030     unsigned long long region_size;
0031     int overlapping;
0032 };
0033
0034 struct test {
0035     const char *name;
0036     struct config config;
0037     int expect_failure;
0038 };
0039
0040 enum {
0041     _1KB = 1ULL << 10,  /* 1KB -> not page aligned */
0042     _4KB = 4ULL << 10,
0043     _8KB = 8ULL << 10,
0044     _1MB = 1ULL << 20,
0045     _2MB = 2ULL << 20,
0046     _4MB = 4ULL << 20,
0047     _1GB = 1ULL << 30,
0048     _2GB = 2ULL << 30,
0049     PMD = _2MB,
0050     PUD = _1GB,
0051 };
0052
0053 #define PTE page_size
0054
0055 #define MAKE_TEST(source_align, destination_align, size,    \
0056           overlaps, should_fail, test_name)     \
0057 (struct test){                          \
0058     .name = test_name,                  \
0059     .config = {                     \
0060         .src_alignment = source_align,          \
0061         .dest_alignment = destination_align,        \
0062         .region_size = size,                \
0063         .overlapping = overlaps,            \
0064     },                          \
0065     .expect_failure = should_fail               \
0066 }
0067
0068 /*
0069  * Returns false if the requested remap region overlaps with an
0070  * existing mapping (e.g text, stack) else returns true.
0071  */
0072 static bool is_remap_region_valid(void *addr, unsigned long long size)
0073 {
0074     void *remap_addr = NULL;
0075     bool ret = true;
0076
0077     /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */
0078     remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE,
0079                      MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
0080                      -1, 0);
0081
0082     if (remap_addr == MAP_FAILED) {
0083         if (errno == EEXIST)
0084             ret = false;
0085     } else {
0086         munmap(remap_addr, size);
0087     }
0088
0089     return ret;
0090 }
0091
0092 /* Returns mmap_min_addr sysctl tunable from procfs */
0093 static unsigned long long get_mmap_min_addr(void)
0094 {
0095     FILE *fp;
0096     int n_matched;
0097     static unsigned long long addr;
0098
0099     if (addr)
0100         return addr;
0101
0102     fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
0103     if (fp == NULL) {
0104         ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
0105             strerror(errno));
0106         exit(KSFT_SKIP);
0107     }
0108
0109     n_matched = fscanf(fp, "%llu", &addr);
0110     if (n_matched != 1) {
0111         ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
0112             strerror(errno));
0113         fclose(fp);
0114         exit(KSFT_SKIP);
0115     }
0116
0117     fclose(fp);
0118     return addr;
0119 }
0120
0121 /*
0122  * Returns the start address of the mapping on success, else returns
0123  * NULL on failure.
0124  */
0125 static void *get_source_mapping(struct config c)
0126 {
0127     unsigned long long addr = 0ULL;
0128     void *src_addr = NULL;
0129     unsigned long long mmap_min_addr;
0130
0131     mmap_min_addr = get_mmap_min_addr();
0132
0133 retry:
0134     addr += c.src_alignment;
0135     if (addr < mmap_min_addr)
0136         goto retry;
0137
0138     src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
0139                     MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
0140                     -1, 0);
0141     if (src_addr == MAP_FAILED) {
0142         if (errno == EPERM || errno == EEXIST)
0143             goto retry;
0144         goto error;
0145     }
0146     /*
0147      * Check that the address is aligned to the specified alignment.
0148      * Addresses which have alignments that are multiples of that
0149      * specified are not considered valid. For instance, 1GB address is
0150      * 2MB-aligned, however it will not be considered valid for a
0151      * requested alignment of 2MB. This is done to reduce coincidental
0152      * alignment in the tests.
0153      */
0154     if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
0155             !((unsigned long long) src_addr & c.src_alignment)) {
0156         munmap(src_addr, c.region_size);
0157         goto retry;
0158     }
0159
0160     if (!src_addr)
0161         goto error;
0162
0163     return src_addr;
0164 error:
0165     ksft_print_msg("Failed to map source region: %s\n",
0166             strerror(errno));
0167     return NULL;
0168 }
0169
0170 /* Returns the time taken for the remap on success else returns -1. */
0171 static long long remap_region(struct config c, unsigned int threshold_mb,
0172                   char pattern_seed)
0173 {
0174     void *addr, *src_addr, *dest_addr;
0175     unsigned long long i;
0176     struct timespec t_start = {0, 0}, t_end = {0, 0};
0177     long long  start_ns, end_ns, align_mask, ret, offset;
0178     unsigned long long threshold;
0179
0180     if (threshold_mb == VALIDATION_NO_THRESHOLD)
0181         threshold = c.region_size;
0182     else
0183         threshold = MIN(threshold_mb * _1MB, c.region_size);
0184
0185     src_addr = get_source_mapping(c);
0186     if (!src_addr) {
0187         ret = -1;
0188         goto out;
0189     }
0190
0191     /* Set byte pattern */
0192     srand(pattern_seed);
0193     for (i = 0; i < threshold; i++)
0194         memset((char *) src_addr + i, (char) rand(), 1);
0195
0196     /* Mask to zero out lower bits of address for alignment */
0197     align_mask = ~(c.dest_alignment - 1);
0198     /* Offset of destination address from the end of the source region */
0199     offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
0200     addr = (void *) (((unsigned long long) src_addr + c.region_size
0201               + offset) & align_mask);
0202
0203     /* See comment in get_source_mapping() */
0204     if (!((unsigned long long) addr & c.dest_alignment))
0205         addr = (void *) ((unsigned long long) addr | c.dest_alignment);
0206
0207     /* Don't destroy existing mappings unless expected to overlap */
0208     while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) {
0209         /* Check for unsigned overflow */
0210         if (addr + c.dest_alignment < addr) {
0211             ksft_print_msg("Couldn't find a valid region to remap to\n");
0212             ret = -1;
0213             goto out;
0214         }
0215         addr += c.dest_alignment;
0216     }
0217
0218     clock_gettime(CLOCK_MONOTONIC, &t_start);
0219     dest_addr = mremap(src_addr, c.region_size, c.region_size,
0220                       MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
0221     clock_gettime(CLOCK_MONOTONIC, &t_end);
0222
0223     if (dest_addr == MAP_FAILED) {
0224         ksft_print_msg("mremap failed: %s\n", strerror(errno));
0225         ret = -1;
0226         goto clean_up_src;
0227     }
0228
0229     /* Verify byte pattern after remapping */
0230     srand(pattern_seed);
0231     for (i = 0; i < threshold; i++) {
0232         char c = (char) rand();
0233
0234         if (((char *) dest_addr)[i] != c) {
0235             ksft_print_msg("Data after remap doesn't match at offset %d\n",
0236                        i);
0237             ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
0238                     ((char *) dest_addr)[i] & 0xff);
0239             ret = -1;
0240             goto clean_up_dest;
0241         }
0242     }
0243
0244     start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
0245     end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
0246     ret = end_ns - start_ns;
0247
0248 /*
0249  * Since the destination address is specified using MREMAP_FIXED, subsequent
0250  * mremap will unmap any previous mapping at the address range specified by
0251  * dest_addr and region_size. This significantly affects the remap time of
0252  * subsequent tests. So we clean up mappings after each test.
0253  */
0254 clean_up_dest:
0255     munmap(dest_addr, c.region_size);
0256 clean_up_src:
0257     munmap(src_addr, c.region_size);
0258 out:
0259     return ret;
0260 }
0261
0262 static void run_mremap_test_case(struct test test_case, int *failures,
0263                  unsigned int threshold_mb,
0264                  unsigned int pattern_seed)
0265 {
0266     long long remap_time = remap_region(test_case.config, threshold_mb,
0267                         pattern_seed);
0268
0269     if (remap_time < 0) {
0270         if (test_case.expect_failure)
0271             ksft_test_result_xfail("%s\n\tExpected mremap failure\n",
0272                           test_case.name);
0273         else {
0274             ksft_test_result_fail("%s\n", test_case.name);
0275             *failures += 1;
0276         }
0277     } else {
0278         /*
0279          * Comparing mremap time is only applicable if entire region
0280          * was faulted in.
0281          */
0282         if (threshold_mb == VALIDATION_NO_THRESHOLD ||
0283             test_case.config.region_size <= threshold_mb * _1MB)
0284             ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
0285                           test_case.name, remap_time);
0286         else
0287             ksft_test_result_pass("%s\n", test_case.name);
0288     }
0289 }
0290
0291 static void usage(const char *cmd)
0292 {
0293     fprintf(stderr,
0294         "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
0295         "-t\t only validate threshold_mb of the remapped region\n"
0296         "  \t if 0 is supplied no threshold is used; all tests\n"
0297         "  \t are run and remapped regions validated fully.\n"
0298         "  \t The default threshold used is 4MB.\n"
0299         "-p\t provide a seed to generate the random pattern for\n"
0300         "  \t validating the remapped region.\n", cmd);
0301 }
0302
0303 static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
0304               unsigned int *pattern_seed)
0305 {
0306     const char *optstr = "t:p:";
0307     int opt;
0308
0309     while ((opt = getopt(argc, argv, optstr)) != -1) {
0310         switch (opt) {
0311         case 't':
0312             *threshold_mb = atoi(optarg);
0313             break;
0314         case 'p':
0315             *pattern_seed = atoi(optarg);
0316             break;
0317         default:
0318             usage(argv[0]);
0319             return -1;
0320         }
0321     }
0322
0323     if (optind < argc) {
0324         usage(argv[0]);
0325         return -1;
0326     }
0327
0328     return 0;
0329 }
0330
0331 #define MAX_TEST 13
0332 #define MAX_PERF_TEST 3
0333 int main(int argc, char **argv)
0334 {
0335     int failures = 0;
0336     int i, run_perf_tests;
0337     unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
0338     unsigned int pattern_seed;
0339     struct test test_cases[MAX_TEST];
0340     struct test perf_test_cases[MAX_PERF_TEST];
0341     int page_size;
0342     time_t t;
0343
0344     pattern_seed = (unsigned int) time(&t);
0345
0346     if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
0347         exit(EXIT_FAILURE);
0348
0349     ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
0350                threshold_mb, pattern_seed);
0351
0352     page_size = sysconf(_SC_PAGESIZE);
0353
0354     /* Expected mremap failures */
0355     test_cases[0] = MAKE_TEST(page_size, page_size, page_size,
0356                   OVERLAPPING, EXPECT_FAILURE,
0357                   "mremap - Source and Destination Regions Overlapping");
0358
0359     test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
0360                   NON_OVERLAPPING, EXPECT_FAILURE,
0361                   "mremap - Destination Address Misaligned (1KB-aligned)");
0362     test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
0363                   NON_OVERLAPPING, EXPECT_FAILURE,
0364                   "mremap - Source Address Misaligned (1KB-aligned)");
0365
0366     /* Src addr PTE aligned */
0367     test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
0368                   NON_OVERLAPPING, EXPECT_SUCCESS,
0369                   "8KB mremap - Source PTE-aligned, Destination PTE-aligned");
0370
0371     /* Src addr 1MB aligned */
0372     test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
0373                   "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
0374     test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
0375                   "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
0376
0377     /* Src addr PMD aligned */
0378     test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
0379                   "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
0380     test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
0381                   "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
0382     test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
0383                   "4MB mremap - Source PMD-aligned, Destination PMD-aligned");
0384
0385     /* Src addr PUD aligned */
0386     test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
0387                   "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
0388     test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
0389                    "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
0390     test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
0391                    "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
0392     test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
0393                    "2GB mremap - Source PUD-aligned, Destination PUD-aligned");
0394
0395     perf_test_cases[0] =  MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
0396                     "1GB mremap - Source PTE-aligned, Destination PTE-aligned");
0397     /*
0398      * mremap 1GB region - Page table level aligned time
0399      * comparison.
0400      */
0401     perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
0402                        "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
0403     perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
0404                        "1GB mremap - Source PUD-aligned, Destination PUD-aligned");
0405
0406     run_perf_tests =  (threshold_mb == VALIDATION_NO_THRESHOLD) ||
0407                 (threshold_mb * _1MB >= _1GB);
0408
0409     ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
0410               ARRAY_SIZE(perf_test_cases) : 0));
0411
0412     for (i = 0; i < ARRAY_SIZE(test_cases); i++)
0413         run_mremap_test_case(test_cases[i], &failures, threshold_mb,
0414                      pattern_seed);
0415
0416     if (run_perf_tests) {
0417         ksft_print_msg("\n%s\n",
0418          "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
0419         for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
0420             run_mremap_test_case(perf_test_cases[i], &failures,
0421                          threshold_mb, pattern_seed);
0422     }
0423
0424     if (failures > 0)
0425         ksft_exit_fail();
0426     else
0427         ksft_exit_pass();
0428 }