arm64/abi/syscall-abi.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2021 ARM Limited.
0004  */
0005
0006 #include <errno.h>
0007 #include <stdbool.h>
0008 #include <stddef.h>
0009 #include <stdio.h>
0010 #include <stdlib.h>
0011 #include <string.h>
0012 #include <unistd.h>
0013 #include <sys/auxv.h>
0014 #include <sys/prctl.h>
0015 #include <asm/hwcap.h>
0016 #include <asm/sigcontext.h>
0017 #include <asm/unistd.h>
0018
0019 #include "../../kselftest.h"
0020
0021 #include "syscall-abi.h"
0022
0023 #define NUM_VL ((SVE_VQ_MAX - SVE_VQ_MIN) + 1)
0024
0025 static int default_sme_vl;
0026
0027 extern void do_syscall(int sve_vl, int sme_vl);
0028
0029 static void fill_random(void *buf, size_t size)
0030 {
0031     int i;
0032     uint32_t *lbuf = buf;
0033
0034     /* random() returns a 32 bit number regardless of the size of long */
0035     for (i = 0; i < size / sizeof(uint32_t); i++)
0036         lbuf[i] = random();
0037 }
0038
0039 /*
0040  * We also repeat the test for several syscalls to try to expose different
0041  * behaviour.
0042  */
0043 static struct syscall_cfg {
0044     int syscall_nr;
0045     const char *name;
0046 } syscalls[] = {
0047     { __NR_getpid,      "getpid()" },
0048     { __NR_sched_yield, "sched_yield()" },
0049 };
0050
0051 #define NUM_GPR 31
0052 uint64_t gpr_in[NUM_GPR];
0053 uint64_t gpr_out[NUM_GPR];
0054
0055 static void setup_gpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0056               uint64_t svcr)
0057 {
0058     fill_random(gpr_in, sizeof(gpr_in));
0059     gpr_in[8] = cfg->syscall_nr;
0060     memset(gpr_out, 0, sizeof(gpr_out));
0061 }
0062
0063 static int check_gpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, uint64_t svcr)
0064 {
0065     int errors = 0;
0066     int i;
0067
0068     /*
0069      * GPR x0-x7 may be clobbered, and all others should be preserved.
0070      */
0071     for (i = 9; i < ARRAY_SIZE(gpr_in); i++) {
0072         if (gpr_in[i] != gpr_out[i]) {
0073             ksft_print_msg("%s SVE VL %d mismatch in GPR %d: %llx != %llx\n",
0074                        cfg->name, sve_vl, i,
0075                        gpr_in[i], gpr_out[i]);
0076             errors++;
0077         }
0078     }
0079
0080     return errors;
0081 }
0082
0083 #define NUM_FPR 32
0084 uint64_t fpr_in[NUM_FPR * 2];
0085 uint64_t fpr_out[NUM_FPR * 2];
0086
0087 static void setup_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0088               uint64_t svcr)
0089 {
0090     fill_random(fpr_in, sizeof(fpr_in));
0091     memset(fpr_out, 0, sizeof(fpr_out));
0092 }
0093
0094 static int check_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0095              uint64_t svcr)
0096 {
0097     int errors = 0;
0098     int i;
0099
0100     if (!sve_vl) {
0101         for (i = 0; i < ARRAY_SIZE(fpr_in); i++) {
0102             if (fpr_in[i] != fpr_out[i]) {
0103                 ksft_print_msg("%s Q%d/%d mismatch %llx != %llx\n",
0104                            cfg->name,
0105                            i / 2, i % 2,
0106                            fpr_in[i], fpr_out[i]);
0107                 errors++;
0108             }
0109         }
0110     }
0111
0112     return errors;
0113 }
0114
0115 static uint8_t z_zero[__SVE_ZREG_SIZE(SVE_VQ_MAX)];
0116 uint8_t z_in[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
0117 uint8_t z_out[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
0118
0119 static void setup_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0120             uint64_t svcr)
0121 {
0122     fill_random(z_in, sizeof(z_in));
0123     fill_random(z_out, sizeof(z_out));
0124 }
0125
0126 static int check_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0127            uint64_t svcr)
0128 {
0129     size_t reg_size = sve_vl;
0130     int errors = 0;
0131     int i;
0132
0133     if (!sve_vl)
0134         return 0;
0135
0136     /*
0137      * After a syscall the low 128 bits of the Z registers should
0138      * be preserved and the rest be zeroed or preserved, except if
0139      * we were in streaming mode in which case the low 128 bits may
0140      * also be cleared by the transition out of streaming mode.
0141      */
0142     for (i = 0; i < SVE_NUM_ZREGS; i++) {
0143         void *in = &z_in[reg_size * i];
0144         void *out = &z_out[reg_size * i];
0145
0146         if ((memcmp(in, out, SVE_VQ_BYTES) != 0) &&
0147             !((svcr & SVCR_SM_MASK) &&
0148               memcmp(z_zero, out, SVE_VQ_BYTES) == 0)) {
0149             ksft_print_msg("%s SVE VL %d Z%d low 128 bits changed\n",
0150                        cfg->name, sve_vl, i);
0151             errors++;
0152         }
0153     }
0154
0155     return errors;
0156 }
0157
0158 uint8_t p_in[SVE_NUM_PREGS * __SVE_PREG_SIZE(SVE_VQ_MAX)];
0159 uint8_t p_out[SVE_NUM_PREGS * __SVE_PREG_SIZE(SVE_VQ_MAX)];
0160
0161 static void setup_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0162             uint64_t svcr)
0163 {
0164     fill_random(p_in, sizeof(p_in));
0165     fill_random(p_out, sizeof(p_out));
0166 }
0167
0168 static int check_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0169            uint64_t svcr)
0170 {
0171     size_t reg_size = sve_vq_from_vl(sve_vl) * 2; /* 1 bit per VL byte */
0172
0173     int errors = 0;
0174     int i;
0175
0176     if (!sve_vl)
0177         return 0;
0178
0179     /* After a syscall the P registers should be preserved or zeroed */
0180     for (i = 0; i < SVE_NUM_PREGS * reg_size; i++)
0181         if (p_out[i] && (p_in[i] != p_out[i]))
0182             errors++;
0183     if (errors)
0184         ksft_print_msg("%s SVE VL %d predicate registers non-zero\n",
0185                    cfg->name, sve_vl);
0186
0187     return errors;
0188 }
0189
0190 uint8_t ffr_in[__SVE_PREG_SIZE(SVE_VQ_MAX)];
0191 uint8_t ffr_out[__SVE_PREG_SIZE(SVE_VQ_MAX)];
0192
0193 static void setup_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0194               uint64_t svcr)
0195 {
0196     /*
0197      * If we are in streaming mode and do not have FA64 then FFR
0198      * is unavailable.
0199      */
0200     if ((svcr & SVCR_SM_MASK) &&
0201         !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64)) {
0202         memset(&ffr_in, 0, sizeof(ffr_in));
0203         return;
0204     }
0205
0206     /*
0207      * It is only valid to set a contiguous set of bits starting
0208      * at 0.  For now since we're expecting this to be cleared by
0209      * a syscall just set all bits.
0210      */
0211     memset(ffr_in, 0xff, sizeof(ffr_in));
0212     fill_random(ffr_out, sizeof(ffr_out));
0213 }
0214
0215 static int check_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0216              uint64_t svcr)
0217 {
0218     size_t reg_size = sve_vq_from_vl(sve_vl) * 2;  /* 1 bit per VL byte */
0219     int errors = 0;
0220     int i;
0221
0222     if (!sve_vl)
0223         return 0;
0224
0225     if ((svcr & SVCR_SM_MASK) &&
0226         !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64))
0227         return 0;
0228
0229     /* After a syscall the P registers should be preserved or zeroed */
0230     for (i = 0; i < reg_size; i++)
0231         if (ffr_out[i] && (ffr_in[i] != ffr_out[i]))
0232             errors++;
0233     if (errors)
0234         ksft_print_msg("%s SVE VL %d FFR non-zero\n",
0235                    cfg->name, sve_vl);
0236
0237     return errors;
0238 }
0239
0240 uint64_t svcr_in, svcr_out;
0241
0242 static void setup_svcr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0243             uint64_t svcr)
0244 {
0245     svcr_in = svcr;
0246 }
0247
0248 static int check_svcr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0249               uint64_t svcr)
0250 {
0251     int errors = 0;
0252
0253     if (svcr_out & SVCR_SM_MASK) {
0254         ksft_print_msg("%s Still in SM, SVCR %llx\n",
0255                    cfg->name, svcr_out);
0256         errors++;
0257     }
0258
0259     if ((svcr_in & SVCR_ZA_MASK) != (svcr_out & SVCR_ZA_MASK)) {
0260         ksft_print_msg("%s PSTATE.ZA changed, SVCR %llx != %llx\n",
0261                    cfg->name, svcr_in, svcr_out);
0262         errors++;
0263     }
0264
0265     return errors;
0266 }
0267
0268 uint8_t za_in[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
0269 uint8_t za_out[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
0270
0271 static void setup_za(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0272              uint64_t svcr)
0273 {
0274     fill_random(za_in, sizeof(za_in));
0275     memset(za_out, 0, sizeof(za_out));
0276 }
0277
0278 static int check_za(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0279             uint64_t svcr)
0280 {
0281     size_t reg_size = sme_vl * sme_vl;
0282     int errors = 0;
0283
0284     if (!(svcr & SVCR_ZA_MASK))
0285         return 0;
0286
0287     if (memcmp(za_in, za_out, reg_size) != 0) {
0288         ksft_print_msg("SME VL %d ZA does not match\n", sme_vl);
0289         errors++;
0290     }
0291
0292     return errors;
0293 }
0294
0295 typedef void (*setup_fn)(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0296              uint64_t svcr);
0297 typedef int (*check_fn)(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0298             uint64_t svcr);
0299
0300 /*
0301  * Each set of registers has a setup function which is called before
0302  * the syscall to fill values in a global variable for loading by the
0303  * test code and a check function which validates that the results are
0304  * as expected.  Vector lengths are passed everywhere, a vector length
0305  * of 0 should be treated as do not test.
0306  */
0307 static struct {
0308     setup_fn setup;
0309     check_fn check;
0310 } regset[] = {
0311     { setup_gpr, check_gpr },
0312     { setup_fpr, check_fpr },
0313     { setup_z, check_z },
0314     { setup_p, check_p },
0315     { setup_ffr, check_ffr },
0316     { setup_svcr, check_svcr },
0317     { setup_za, check_za },
0318 };
0319
0320 static bool do_test(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
0321             uint64_t svcr)
0322 {
0323     int errors = 0;
0324     int i;
0325
0326     for (i = 0; i < ARRAY_SIZE(regset); i++)
0327         regset[i].setup(cfg, sve_vl, sme_vl, svcr);
0328
0329     do_syscall(sve_vl, sme_vl);
0330
0331     for (i = 0; i < ARRAY_SIZE(regset); i++)
0332         errors += regset[i].check(cfg, sve_vl, sme_vl, svcr);
0333
0334     return errors == 0;
0335 }
0336
0337 static void test_one_syscall(struct syscall_cfg *cfg)
0338 {
0339     int sve_vq, sve_vl;
0340     int sme_vq, sme_vl;
0341
0342     /* FPSIMD only case */
0343     ksft_test_result(do_test(cfg, 0, default_sme_vl, 0),
0344              "%s FPSIMD\n", cfg->name);
0345
0346     if (!(getauxval(AT_HWCAP) & HWCAP_SVE))
0347         return;
0348
0349     for (sve_vq = SVE_VQ_MAX; sve_vq > 0; --sve_vq) {
0350         sve_vl = prctl(PR_SVE_SET_VL, sve_vq * 16);
0351         if (sve_vl == -1)
0352             ksft_exit_fail_msg("PR_SVE_SET_VL failed: %s (%d)\n",
0353                        strerror(errno), errno);
0354
0355         sve_vl &= PR_SVE_VL_LEN_MASK;
0356
0357         if (sve_vq != sve_vq_from_vl(sve_vl))
0358             sve_vq = sve_vq_from_vl(sve_vl);
0359
0360         ksft_test_result(do_test(cfg, sve_vl, default_sme_vl, 0),
0361                  "%s SVE VL %d\n", cfg->name, sve_vl);
0362
0363         if (!(getauxval(AT_HWCAP2) & HWCAP2_SME))
0364             continue;
0365
0366         for (sme_vq = SVE_VQ_MAX; sme_vq > 0; --sme_vq) {
0367             sme_vl = prctl(PR_SME_SET_VL, sme_vq * 16);
0368             if (sme_vl == -1)
0369                 ksft_exit_fail_msg("PR_SME_SET_VL failed: %s (%d)\n",
0370                            strerror(errno), errno);
0371
0372             sme_vl &= PR_SME_VL_LEN_MASK;
0373
0374             if (sme_vq != sve_vq_from_vl(sme_vl))
0375                 sme_vq = sve_vq_from_vl(sme_vl);
0376
0377             ksft_test_result(do_test(cfg, sve_vl, sme_vl,
0378                          SVCR_ZA_MASK | SVCR_SM_MASK),
0379                      "%s SVE VL %d/SME VL %d SM+ZA\n",
0380                      cfg->name, sve_vl, sme_vl);
0381             ksft_test_result(do_test(cfg, sve_vl, sme_vl,
0382                          SVCR_SM_MASK),
0383                      "%s SVE VL %d/SME VL %d SM\n",
0384                      cfg->name, sve_vl, sme_vl);
0385             ksft_test_result(do_test(cfg, sve_vl, sme_vl,
0386                          SVCR_ZA_MASK),
0387                      "%s SVE VL %d/SME VL %d ZA\n",
0388                      cfg->name, sve_vl, sme_vl);
0389         }
0390     }
0391 }
0392
0393 int sve_count_vls(void)
0394 {
0395     unsigned int vq;
0396     int vl_count = 0;
0397     int vl;
0398
0399     if (!(getauxval(AT_HWCAP) & HWCAP_SVE))
0400         return 0;
0401
0402     /*
0403      * Enumerate up to SVE_VQ_MAX vector lengths
0404      */
0405     for (vq = SVE_VQ_MAX; vq > 0; --vq) {
0406         vl = prctl(PR_SVE_SET_VL, vq * 16);
0407         if (vl == -1)
0408             ksft_exit_fail_msg("PR_SVE_SET_VL failed: %s (%d)\n",
0409                        strerror(errno), errno);
0410
0411         vl &= PR_SVE_VL_LEN_MASK;
0412
0413         if (vq != sve_vq_from_vl(vl))
0414             vq = sve_vq_from_vl(vl);
0415
0416         vl_count++;
0417     }
0418
0419     return vl_count;
0420 }
0421
0422 int sme_count_vls(void)
0423 {
0424     unsigned int vq;
0425     int vl_count = 0;
0426     int vl;
0427
0428     if (!(getauxval(AT_HWCAP2) & HWCAP2_SME))
0429         return 0;
0430
0431     /* Ensure we configure a SME VL, used to flag if SVCR is set */
0432     default_sme_vl = 16;
0433
0434     /*
0435      * Enumerate up to SVE_VQ_MAX vector lengths
0436      */
0437     for (vq = SVE_VQ_MAX; vq > 0; --vq) {
0438         vl = prctl(PR_SME_SET_VL, vq * 16);
0439         if (vl == -1)
0440             ksft_exit_fail_msg("PR_SME_SET_VL failed: %s (%d)\n",
0441                        strerror(errno), errno);
0442
0443         vl &= PR_SME_VL_LEN_MASK;
0444
0445         if (vq != sve_vq_from_vl(vl))
0446             vq = sve_vq_from_vl(vl);
0447
0448         vl_count++;
0449     }
0450
0451     return vl_count;
0452 }
0453
0454 int main(void)
0455 {
0456     int i;
0457     int tests = 1;  /* FPSIMD */
0458
0459     srandom(getpid());
0460
0461     ksft_print_header();
0462     tests += sve_count_vls();
0463     tests += (sve_count_vls() * sme_count_vls()) * 3;
0464     ksft_set_plan(ARRAY_SIZE(syscalls) * tests);
0465
0466     if (getauxval(AT_HWCAP2) & HWCAP2_SME_FA64)
0467         ksft_print_msg("SME with FA64\n");
0468     else if (getauxval(AT_HWCAP2) & HWCAP2_SME)
0469         ksft_print_msg("SME without FA64\n");
0470
0471     for (i = 0; i < ARRAY_SIZE(syscalls); i++)
0472         test_one_syscall(&syscalls[i]);
0473
0474     ksft_print_cnts();
0475
0476     return 0;
0477 }