Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2012,2013 - ARM Ltd
0004  * Author: Marc Zyngier <marc.zyngier@arm.com>
0005  *
0006  * Derived from arch/arm/kvm/guest.c:
0007  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
0008  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
0009  */
0010 
0011 #include <linux/bits.h>
0012 #include <linux/errno.h>
0013 #include <linux/err.h>
0014 #include <linux/nospec.h>
0015 #include <linux/kvm_host.h>
0016 #include <linux/module.h>
0017 #include <linux/stddef.h>
0018 #include <linux/string.h>
0019 #include <linux/vmalloc.h>
0020 #include <linux/fs.h>
0021 #include <kvm/arm_hypercalls.h>
0022 #include <asm/cputype.h>
0023 #include <linux/uaccess.h>
0024 #include <asm/fpsimd.h>
0025 #include <asm/kvm.h>
0026 #include <asm/kvm_emulate.h>
0027 #include <asm/sigcontext.h>
0028 
0029 #include "trace.h"
0030 
0031 const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
0032     KVM_GENERIC_VM_STATS()
0033 };
0034 
0035 const struct kvm_stats_header kvm_vm_stats_header = {
0036     .name_size = KVM_STATS_NAME_SIZE,
0037     .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
0038     .id_offset =  sizeof(struct kvm_stats_header),
0039     .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
0040     .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
0041                sizeof(kvm_vm_stats_desc),
0042 };
0043 
0044 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
0045     KVM_GENERIC_VCPU_STATS(),
0046     STATS_DESC_COUNTER(VCPU, hvc_exit_stat),
0047     STATS_DESC_COUNTER(VCPU, wfe_exit_stat),
0048     STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
0049     STATS_DESC_COUNTER(VCPU, mmio_exit_user),
0050     STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
0051     STATS_DESC_COUNTER(VCPU, signal_exits),
0052     STATS_DESC_COUNTER(VCPU, exits)
0053 };
0054 
0055 const struct kvm_stats_header kvm_vcpu_stats_header = {
0056     .name_size = KVM_STATS_NAME_SIZE,
0057     .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
0058     .id_offset = sizeof(struct kvm_stats_header),
0059     .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
0060     .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
0061                sizeof(kvm_vcpu_stats_desc),
0062 };
0063 
0064 static bool core_reg_offset_is_vreg(u64 off)
0065 {
0066     return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) &&
0067         off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr);
0068 }
0069 
0070 static u64 core_reg_offset_from_id(u64 id)
0071 {
0072     return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE);
0073 }
0074 
0075 static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off)
0076 {
0077     int size;
0078 
0079     switch (off) {
0080     case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
0081          KVM_REG_ARM_CORE_REG(regs.regs[30]):
0082     case KVM_REG_ARM_CORE_REG(regs.sp):
0083     case KVM_REG_ARM_CORE_REG(regs.pc):
0084     case KVM_REG_ARM_CORE_REG(regs.pstate):
0085     case KVM_REG_ARM_CORE_REG(sp_el1):
0086     case KVM_REG_ARM_CORE_REG(elr_el1):
0087     case KVM_REG_ARM_CORE_REG(spsr[0]) ...
0088          KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]):
0089         size = sizeof(__u64);
0090         break;
0091 
0092     case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
0093          KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
0094         size = sizeof(__uint128_t);
0095         break;
0096 
0097     case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
0098     case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
0099         size = sizeof(__u32);
0100         break;
0101 
0102     default:
0103         return -EINVAL;
0104     }
0105 
0106     if (!IS_ALIGNED(off, size / sizeof(__u32)))
0107         return -EINVAL;
0108 
0109     /*
0110      * The KVM_REG_ARM64_SVE regs must be used instead of
0111      * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on
0112      * SVE-enabled vcpus:
0113      */
0114     if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off))
0115         return -EINVAL;
0116 
0117     return size;
0118 }
0119 
0120 static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0121 {
0122     u64 off = core_reg_offset_from_id(reg->id);
0123     int size = core_reg_size_from_offset(vcpu, off);
0124 
0125     if (size < 0)
0126         return NULL;
0127 
0128     if (KVM_REG_SIZE(reg->id) != size)
0129         return NULL;
0130 
0131     switch (off) {
0132     case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
0133          KVM_REG_ARM_CORE_REG(regs.regs[30]):
0134         off -= KVM_REG_ARM_CORE_REG(regs.regs[0]);
0135         off /= 2;
0136         return &vcpu->arch.ctxt.regs.regs[off];
0137 
0138     case KVM_REG_ARM_CORE_REG(regs.sp):
0139         return &vcpu->arch.ctxt.regs.sp;
0140 
0141     case KVM_REG_ARM_CORE_REG(regs.pc):
0142         return &vcpu->arch.ctxt.regs.pc;
0143 
0144     case KVM_REG_ARM_CORE_REG(regs.pstate):
0145         return &vcpu->arch.ctxt.regs.pstate;
0146 
0147     case KVM_REG_ARM_CORE_REG(sp_el1):
0148         return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1);
0149 
0150     case KVM_REG_ARM_CORE_REG(elr_el1):
0151         return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1);
0152 
0153     case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_EL1]):
0154         return __ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1);
0155 
0156     case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_ABT]):
0157         return &vcpu->arch.ctxt.spsr_abt;
0158 
0159     case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_UND]):
0160         return &vcpu->arch.ctxt.spsr_und;
0161 
0162     case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_IRQ]):
0163         return &vcpu->arch.ctxt.spsr_irq;
0164 
0165     case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_FIQ]):
0166         return &vcpu->arch.ctxt.spsr_fiq;
0167 
0168     case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
0169          KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
0170         off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]);
0171         off /= 4;
0172         return &vcpu->arch.ctxt.fp_regs.vregs[off];
0173 
0174     case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
0175         return &vcpu->arch.ctxt.fp_regs.fpsr;
0176 
0177     case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
0178         return &vcpu->arch.ctxt.fp_regs.fpcr;
0179 
0180     default:
0181         return NULL;
0182     }
0183 }
0184 
0185 static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0186 {
0187     /*
0188      * Because the kvm_regs structure is a mix of 32, 64 and
0189      * 128bit fields, we index it as if it was a 32bit
0190      * array. Hence below, nr_regs is the number of entries, and
0191      * off the index in the "array".
0192      */
0193     __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
0194     int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32);
0195     void *addr;
0196     u32 off;
0197 
0198     /* Our ID is an index into the kvm_regs struct. */
0199     off = core_reg_offset_from_id(reg->id);
0200     if (off >= nr_regs ||
0201         (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
0202         return -ENOENT;
0203 
0204     addr = core_reg_addr(vcpu, reg);
0205     if (!addr)
0206         return -EINVAL;
0207 
0208     if (copy_to_user(uaddr, addr, KVM_REG_SIZE(reg->id)))
0209         return -EFAULT;
0210 
0211     return 0;
0212 }
0213 
0214 static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0215 {
0216     __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
0217     int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32);
0218     __uint128_t tmp;
0219     void *valp = &tmp, *addr;
0220     u64 off;
0221     int err = 0;
0222 
0223     /* Our ID is an index into the kvm_regs struct. */
0224     off = core_reg_offset_from_id(reg->id);
0225     if (off >= nr_regs ||
0226         (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
0227         return -ENOENT;
0228 
0229     addr = core_reg_addr(vcpu, reg);
0230     if (!addr)
0231         return -EINVAL;
0232 
0233     if (KVM_REG_SIZE(reg->id) > sizeof(tmp))
0234         return -EINVAL;
0235 
0236     if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) {
0237         err = -EFAULT;
0238         goto out;
0239     }
0240 
0241     if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) {
0242         u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK;
0243         switch (mode) {
0244         case PSR_AA32_MODE_USR:
0245             if (!kvm_supports_32bit_el0())
0246                 return -EINVAL;
0247             break;
0248         case PSR_AA32_MODE_FIQ:
0249         case PSR_AA32_MODE_IRQ:
0250         case PSR_AA32_MODE_SVC:
0251         case PSR_AA32_MODE_ABT:
0252         case PSR_AA32_MODE_UND:
0253             if (!vcpu_el1_is_32bit(vcpu))
0254                 return -EINVAL;
0255             break;
0256         case PSR_MODE_EL0t:
0257         case PSR_MODE_EL1t:
0258         case PSR_MODE_EL1h:
0259             if (vcpu_el1_is_32bit(vcpu))
0260                 return -EINVAL;
0261             break;
0262         default:
0263             err = -EINVAL;
0264             goto out;
0265         }
0266     }
0267 
0268     memcpy(addr, valp, KVM_REG_SIZE(reg->id));
0269 
0270     if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) {
0271         int i, nr_reg;
0272 
0273         switch (*vcpu_cpsr(vcpu)) {
0274         /*
0275          * Either we are dealing with user mode, and only the
0276          * first 15 registers (+ PC) must be narrowed to 32bit.
0277          * AArch32 r0-r14 conveniently map to AArch64 x0-x14.
0278          */
0279         case PSR_AA32_MODE_USR:
0280         case PSR_AA32_MODE_SYS:
0281             nr_reg = 15;
0282             break;
0283 
0284         /*
0285          * Otherwise, this is a privileged mode, and *all* the
0286          * registers must be narrowed to 32bit.
0287          */
0288         default:
0289             nr_reg = 31;
0290             break;
0291         }
0292 
0293         for (i = 0; i < nr_reg; i++)
0294             vcpu_set_reg(vcpu, i, (u32)vcpu_get_reg(vcpu, i));
0295 
0296         *vcpu_pc(vcpu) = (u32)*vcpu_pc(vcpu);
0297     }
0298 out:
0299     return err;
0300 }
0301 
0302 #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64)
0303 #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64)
0304 #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq)))
0305 
0306 static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0307 {
0308     unsigned int max_vq, vq;
0309     u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
0310 
0311     if (!vcpu_has_sve(vcpu))
0312         return -ENOENT;
0313 
0314     if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl)))
0315         return -EINVAL;
0316 
0317     memset(vqs, 0, sizeof(vqs));
0318 
0319     max_vq = vcpu_sve_max_vq(vcpu);
0320     for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
0321         if (sve_vq_available(vq))
0322             vqs[vq_word(vq)] |= vq_mask(vq);
0323 
0324     if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs)))
0325         return -EFAULT;
0326 
0327     return 0;
0328 }
0329 
0330 static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0331 {
0332     unsigned int max_vq, vq;
0333     u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
0334 
0335     if (!vcpu_has_sve(vcpu))
0336         return -ENOENT;
0337 
0338     if (kvm_arm_vcpu_sve_finalized(vcpu))
0339         return -EPERM; /* too late! */
0340 
0341     if (WARN_ON(vcpu->arch.sve_state))
0342         return -EINVAL;
0343 
0344     if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs)))
0345         return -EFAULT;
0346 
0347     max_vq = 0;
0348     for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq)
0349         if (vq_present(vqs, vq))
0350             max_vq = vq;
0351 
0352     if (max_vq > sve_vq_from_vl(kvm_sve_max_vl))
0353         return -EINVAL;
0354 
0355     /*
0356      * Vector lengths supported by the host can't currently be
0357      * hidden from the guest individually: instead we can only set a
0358      * maximum via ZCR_EL2.LEN.  So, make sure the available vector
0359      * lengths match the set requested exactly up to the requested
0360      * maximum:
0361      */
0362     for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
0363         if (vq_present(vqs, vq) != sve_vq_available(vq))
0364             return -EINVAL;
0365 
0366     /* Can't run with no vector lengths at all: */
0367     if (max_vq < SVE_VQ_MIN)
0368         return -EINVAL;
0369 
0370     /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */
0371     vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq);
0372 
0373     return 0;
0374 }
0375 
0376 #define SVE_REG_SLICE_SHIFT 0
0377 #define SVE_REG_SLICE_BITS  5
0378 #define SVE_REG_ID_SHIFT    (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS)
0379 #define SVE_REG_ID_BITS     5
0380 
0381 #define SVE_REG_SLICE_MASK                  \
0382     GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1,   \
0383         SVE_REG_SLICE_SHIFT)
0384 #define SVE_REG_ID_MASK                         \
0385     GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT)
0386 
0387 #define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS)
0388 
0389 #define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0))
0390 #define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0))
0391 
0392 /*
0393  * Number of register slices required to cover each whole SVE register.
0394  * NOTE: Only the first slice every exists, for now.
0395  * If you are tempted to modify this, you must also rework sve_reg_to_region()
0396  * to match:
0397  */
0398 #define vcpu_sve_slices(vcpu) 1
0399 
0400 /* Bounds of a single SVE register slice within vcpu->arch.sve_state */
0401 struct sve_state_reg_region {
0402     unsigned int koffset;   /* offset into sve_state in kernel memory */
0403     unsigned int klen;  /* length in kernel memory */
0404     unsigned int upad;  /* extra trailing padding in user memory */
0405 };
0406 
0407 /*
0408  * Validate SVE register ID and get sanitised bounds for user/kernel SVE
0409  * register copy
0410  */
0411 static int sve_reg_to_region(struct sve_state_reg_region *region,
0412                  struct kvm_vcpu *vcpu,
0413                  const struct kvm_one_reg *reg)
0414 {
0415     /* reg ID ranges for Z- registers */
0416     const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0);
0417     const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1,
0418                                SVE_NUM_SLICES - 1);
0419 
0420     /* reg ID ranges for P- registers and FFR (which are contiguous) */
0421     const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0);
0422     const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1);
0423 
0424     unsigned int vq;
0425     unsigned int reg_num;
0426 
0427     unsigned int reqoffset, reqlen; /* User-requested offset and length */
0428     unsigned int maxlen; /* Maximum permitted length */
0429 
0430     size_t sve_state_size;
0431 
0432     const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1,
0433                             SVE_NUM_SLICES - 1);
0434 
0435     /* Verify that the P-regs and FFR really do have contiguous IDs: */
0436     BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1);
0437 
0438     /* Verify that we match the UAPI header: */
0439     BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES);
0440 
0441     reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT;
0442 
0443     if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) {
0444         if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
0445             return -ENOENT;
0446 
0447         vq = vcpu_sve_max_vq(vcpu);
0448 
0449         reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) -
0450                 SVE_SIG_REGS_OFFSET;
0451         reqlen = KVM_SVE_ZREG_SIZE;
0452         maxlen = SVE_SIG_ZREG_SIZE(vq);
0453     } else if (reg->id >= preg_id_min && reg->id <= preg_id_max) {
0454         if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
0455             return -ENOENT;
0456 
0457         vq = vcpu_sve_max_vq(vcpu);
0458 
0459         reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) -
0460                 SVE_SIG_REGS_OFFSET;
0461         reqlen = KVM_SVE_PREG_SIZE;
0462         maxlen = SVE_SIG_PREG_SIZE(vq);
0463     } else {
0464         return -EINVAL;
0465     }
0466 
0467     sve_state_size = vcpu_sve_state_size(vcpu);
0468     if (WARN_ON(!sve_state_size))
0469         return -EINVAL;
0470 
0471     region->koffset = array_index_nospec(reqoffset, sve_state_size);
0472     region->klen = min(maxlen, reqlen);
0473     region->upad = reqlen - region->klen;
0474 
0475     return 0;
0476 }
0477 
0478 static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0479 {
0480     int ret;
0481     struct sve_state_reg_region region;
0482     char __user *uptr = (char __user *)reg->addr;
0483 
0484     /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
0485     if (reg->id == KVM_REG_ARM64_SVE_VLS)
0486         return get_sve_vls(vcpu, reg);
0487 
0488     /* Try to interpret reg ID as an architectural SVE register... */
0489     ret = sve_reg_to_region(&region, vcpu, reg);
0490     if (ret)
0491         return ret;
0492 
0493     if (!kvm_arm_vcpu_sve_finalized(vcpu))
0494         return -EPERM;
0495 
0496     if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset,
0497              region.klen) ||
0498         clear_user(uptr + region.klen, region.upad))
0499         return -EFAULT;
0500 
0501     return 0;
0502 }
0503 
0504 static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0505 {
0506     int ret;
0507     struct sve_state_reg_region region;
0508     const char __user *uptr = (const char __user *)reg->addr;
0509 
0510     /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */
0511     if (reg->id == KVM_REG_ARM64_SVE_VLS)
0512         return set_sve_vls(vcpu, reg);
0513 
0514     /* Try to interpret reg ID as an architectural SVE register... */
0515     ret = sve_reg_to_region(&region, vcpu, reg);
0516     if (ret)
0517         return ret;
0518 
0519     if (!kvm_arm_vcpu_sve_finalized(vcpu))
0520         return -EPERM;
0521 
0522     if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr,
0523                region.klen))
0524         return -EFAULT;
0525 
0526     return 0;
0527 }
0528 
0529 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
0530 {
0531     return -EINVAL;
0532 }
0533 
0534 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
0535 {
0536     return -EINVAL;
0537 }
0538 
0539 static int copy_core_reg_indices(const struct kvm_vcpu *vcpu,
0540                  u64 __user *uindices)
0541 {
0542     unsigned int i;
0543     int n = 0;
0544 
0545     for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) {
0546         u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i;
0547         int size = core_reg_size_from_offset(vcpu, i);
0548 
0549         if (size < 0)
0550             continue;
0551 
0552         switch (size) {
0553         case sizeof(__u32):
0554             reg |= KVM_REG_SIZE_U32;
0555             break;
0556 
0557         case sizeof(__u64):
0558             reg |= KVM_REG_SIZE_U64;
0559             break;
0560 
0561         case sizeof(__uint128_t):
0562             reg |= KVM_REG_SIZE_U128;
0563             break;
0564 
0565         default:
0566             WARN_ON(1);
0567             continue;
0568         }
0569 
0570         if (uindices) {
0571             if (put_user(reg, uindices))
0572                 return -EFAULT;
0573             uindices++;
0574         }
0575 
0576         n++;
0577     }
0578 
0579     return n;
0580 }
0581 
0582 static unsigned long num_core_regs(const struct kvm_vcpu *vcpu)
0583 {
0584     return copy_core_reg_indices(vcpu, NULL);
0585 }
0586 
0587 /**
0588  * ARM64 versions of the TIMER registers, always available on arm64
0589  */
0590 
0591 #define NUM_TIMER_REGS 3
0592 
0593 static bool is_timer_reg(u64 index)
0594 {
0595     switch (index) {
0596     case KVM_REG_ARM_TIMER_CTL:
0597     case KVM_REG_ARM_TIMER_CNT:
0598     case KVM_REG_ARM_TIMER_CVAL:
0599         return true;
0600     }
0601     return false;
0602 }
0603 
0604 static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
0605 {
0606     if (put_user(KVM_REG_ARM_TIMER_CTL, uindices))
0607         return -EFAULT;
0608     uindices++;
0609     if (put_user(KVM_REG_ARM_TIMER_CNT, uindices))
0610         return -EFAULT;
0611     uindices++;
0612     if (put_user(KVM_REG_ARM_TIMER_CVAL, uindices))
0613         return -EFAULT;
0614 
0615     return 0;
0616 }
0617 
0618 static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0619 {
0620     void __user *uaddr = (void __user *)(long)reg->addr;
0621     u64 val;
0622     int ret;
0623 
0624     ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id));
0625     if (ret != 0)
0626         return -EFAULT;
0627 
0628     return kvm_arm_timer_set_reg(vcpu, reg->id, val);
0629 }
0630 
0631 static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0632 {
0633     void __user *uaddr = (void __user *)(long)reg->addr;
0634     u64 val;
0635 
0636     val = kvm_arm_timer_get_reg(vcpu, reg->id);
0637     return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0;
0638 }
0639 
0640 static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu)
0641 {
0642     const unsigned int slices = vcpu_sve_slices(vcpu);
0643 
0644     if (!vcpu_has_sve(vcpu))
0645         return 0;
0646 
0647     /* Policed by KVM_GET_REG_LIST: */
0648     WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
0649 
0650     return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */)
0651         + 1; /* KVM_REG_ARM64_SVE_VLS */
0652 }
0653 
0654 static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu,
0655                 u64 __user *uindices)
0656 {
0657     const unsigned int slices = vcpu_sve_slices(vcpu);
0658     u64 reg;
0659     unsigned int i, n;
0660     int num_regs = 0;
0661 
0662     if (!vcpu_has_sve(vcpu))
0663         return 0;
0664 
0665     /* Policed by KVM_GET_REG_LIST: */
0666     WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu));
0667 
0668     /*
0669      * Enumerate this first, so that userspace can save/restore in
0670      * the order reported by KVM_GET_REG_LIST:
0671      */
0672     reg = KVM_REG_ARM64_SVE_VLS;
0673     if (put_user(reg, uindices++))
0674         return -EFAULT;
0675     ++num_regs;
0676 
0677     for (i = 0; i < slices; i++) {
0678         for (n = 0; n < SVE_NUM_ZREGS; n++) {
0679             reg = KVM_REG_ARM64_SVE_ZREG(n, i);
0680             if (put_user(reg, uindices++))
0681                 return -EFAULT;
0682             num_regs++;
0683         }
0684 
0685         for (n = 0; n < SVE_NUM_PREGS; n++) {
0686             reg = KVM_REG_ARM64_SVE_PREG(n, i);
0687             if (put_user(reg, uindices++))
0688                 return -EFAULT;
0689             num_regs++;
0690         }
0691 
0692         reg = KVM_REG_ARM64_SVE_FFR(i);
0693         if (put_user(reg, uindices++))
0694             return -EFAULT;
0695         num_regs++;
0696     }
0697 
0698     return num_regs;
0699 }
0700 
0701 /**
0702  * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG
0703  *
0704  * This is for all registers.
0705  */
0706 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu)
0707 {
0708     unsigned long res = 0;
0709 
0710     res += num_core_regs(vcpu);
0711     res += num_sve_regs(vcpu);
0712     res += kvm_arm_num_sys_reg_descs(vcpu);
0713     res += kvm_arm_get_fw_num_regs(vcpu);
0714     res += NUM_TIMER_REGS;
0715 
0716     return res;
0717 }
0718 
0719 /**
0720  * kvm_arm_copy_reg_indices - get indices of all registers.
0721  *
0722  * We do core registers right here, then we append system regs.
0723  */
0724 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
0725 {
0726     int ret;
0727 
0728     ret = copy_core_reg_indices(vcpu, uindices);
0729     if (ret < 0)
0730         return ret;
0731     uindices += ret;
0732 
0733     ret = copy_sve_reg_indices(vcpu, uindices);
0734     if (ret < 0)
0735         return ret;
0736     uindices += ret;
0737 
0738     ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices);
0739     if (ret < 0)
0740         return ret;
0741     uindices += kvm_arm_get_fw_num_regs(vcpu);
0742 
0743     ret = copy_timer_indices(vcpu, uindices);
0744     if (ret < 0)
0745         return ret;
0746     uindices += NUM_TIMER_REGS;
0747 
0748     return kvm_arm_copy_sys_reg_indices(vcpu, uindices);
0749 }
0750 
0751 int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0752 {
0753     /* We currently use nothing arch-specific in upper 32 bits */
0754     if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
0755         return -EINVAL;
0756 
0757     switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
0758     case KVM_REG_ARM_CORE:  return get_core_reg(vcpu, reg);
0759     case KVM_REG_ARM_FW:
0760     case KVM_REG_ARM_FW_FEAT_BMAP:
0761         return kvm_arm_get_fw_reg(vcpu, reg);
0762     case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg);
0763     }
0764 
0765     if (is_timer_reg(reg->id))
0766         return get_timer_reg(vcpu, reg);
0767 
0768     return kvm_arm_sys_reg_get_reg(vcpu, reg);
0769 }
0770 
0771 int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
0772 {
0773     /* We currently use nothing arch-specific in upper 32 bits */
0774     if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32)
0775         return -EINVAL;
0776 
0777     switch (reg->id & KVM_REG_ARM_COPROC_MASK) {
0778     case KVM_REG_ARM_CORE:  return set_core_reg(vcpu, reg);
0779     case KVM_REG_ARM_FW:
0780     case KVM_REG_ARM_FW_FEAT_BMAP:
0781         return kvm_arm_set_fw_reg(vcpu, reg);
0782     case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg);
0783     }
0784 
0785     if (is_timer_reg(reg->id))
0786         return set_timer_reg(vcpu, reg);
0787 
0788     return kvm_arm_sys_reg_set_reg(vcpu, reg);
0789 }
0790 
0791 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
0792                   struct kvm_sregs *sregs)
0793 {
0794     return -EINVAL;
0795 }
0796 
0797 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
0798                   struct kvm_sregs *sregs)
0799 {
0800     return -EINVAL;
0801 }
0802 
0803 int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
0804                   struct kvm_vcpu_events *events)
0805 {
0806     events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE);
0807     events->exception.serror_has_esr = cpus_have_const_cap(ARM64_HAS_RAS_EXTN);
0808 
0809     if (events->exception.serror_pending && events->exception.serror_has_esr)
0810         events->exception.serror_esr = vcpu_get_vsesr(vcpu);
0811 
0812     /*
0813      * We never return a pending ext_dabt here because we deliver it to
0814      * the virtual CPU directly when setting the event and it's no longer
0815      * 'pending' at this point.
0816      */
0817 
0818     return 0;
0819 }
0820 
0821 int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
0822                   struct kvm_vcpu_events *events)
0823 {
0824     bool serror_pending = events->exception.serror_pending;
0825     bool has_esr = events->exception.serror_has_esr;
0826     bool ext_dabt_pending = events->exception.ext_dabt_pending;
0827 
0828     if (serror_pending && has_esr) {
0829         if (!cpus_have_const_cap(ARM64_HAS_RAS_EXTN))
0830             return -EINVAL;
0831 
0832         if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK))
0833             kvm_set_sei_esr(vcpu, events->exception.serror_esr);
0834         else
0835             return -EINVAL;
0836     } else if (serror_pending) {
0837         kvm_inject_vabt(vcpu);
0838     }
0839 
0840     if (ext_dabt_pending)
0841         kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
0842 
0843     return 0;
0844 }
0845 
0846 u32 __attribute_const__ kvm_target_cpu(void)
0847 {
0848     unsigned long implementor = read_cpuid_implementor();
0849     unsigned long part_number = read_cpuid_part_number();
0850 
0851     switch (implementor) {
0852     case ARM_CPU_IMP_ARM:
0853         switch (part_number) {
0854         case ARM_CPU_PART_AEM_V8:
0855             return KVM_ARM_TARGET_AEM_V8;
0856         case ARM_CPU_PART_FOUNDATION:
0857             return KVM_ARM_TARGET_FOUNDATION_V8;
0858         case ARM_CPU_PART_CORTEX_A53:
0859             return KVM_ARM_TARGET_CORTEX_A53;
0860         case ARM_CPU_PART_CORTEX_A57:
0861             return KVM_ARM_TARGET_CORTEX_A57;
0862         }
0863         break;
0864     case ARM_CPU_IMP_APM:
0865         switch (part_number) {
0866         case APM_CPU_PART_POTENZA:
0867             return KVM_ARM_TARGET_XGENE_POTENZA;
0868         }
0869         break;
0870     }
0871 
0872     /* Return a default generic target */
0873     return KVM_ARM_TARGET_GENERIC_V8;
0874 }
0875 
0876 void kvm_vcpu_preferred_target(struct kvm_vcpu_init *init)
0877 {
0878     u32 target = kvm_target_cpu();
0879 
0880     memset(init, 0, sizeof(*init));
0881 
0882     /*
0883      * For now, we don't return any features.
0884      * In future, we might use features to return target
0885      * specific features available for the preferred
0886      * target type.
0887      */
0888     init->target = (__u32)target;
0889 }
0890 
0891 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
0892 {
0893     return -EINVAL;
0894 }
0895 
0896 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
0897 {
0898     return -EINVAL;
0899 }
0900 
0901 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
0902                   struct kvm_translation *tr)
0903 {
0904     return -EINVAL;
0905 }
0906 
0907 /**
0908  * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging
0909  * @kvm:    pointer to the KVM struct
0910  * @kvm_guest_debug: the ioctl data buffer
0911  *
0912  * This sets up and enables the VM for guest debugging. Userspace
0913  * passes in a control flag to enable different debug types and
0914  * potentially other architecture specific information in the rest of
0915  * the structure.
0916  */
0917 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
0918                     struct kvm_guest_debug *dbg)
0919 {
0920     int ret = 0;
0921 
0922     trace_kvm_set_guest_debug(vcpu, dbg->control);
0923 
0924     if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) {
0925         ret = -EINVAL;
0926         goto out;
0927     }
0928 
0929     if (dbg->control & KVM_GUESTDBG_ENABLE) {
0930         vcpu->guest_debug = dbg->control;
0931 
0932         /* Hardware assisted Break and Watch points */
0933         if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) {
0934             vcpu->arch.external_debug_state = dbg->arch;
0935         }
0936 
0937     } else {
0938         /* If not enabled clear all flags */
0939         vcpu->guest_debug = 0;
0940     }
0941 
0942 out:
0943     return ret;
0944 }
0945 
0946 int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
0947                    struct kvm_device_attr *attr)
0948 {
0949     int ret;
0950 
0951     switch (attr->group) {
0952     case KVM_ARM_VCPU_PMU_V3_CTRL:
0953         ret = kvm_arm_pmu_v3_set_attr(vcpu, attr);
0954         break;
0955     case KVM_ARM_VCPU_TIMER_CTRL:
0956         ret = kvm_arm_timer_set_attr(vcpu, attr);
0957         break;
0958     case KVM_ARM_VCPU_PVTIME_CTRL:
0959         ret = kvm_arm_pvtime_set_attr(vcpu, attr);
0960         break;
0961     default:
0962         ret = -ENXIO;
0963         break;
0964     }
0965 
0966     return ret;
0967 }
0968 
0969 int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
0970                    struct kvm_device_attr *attr)
0971 {
0972     int ret;
0973 
0974     switch (attr->group) {
0975     case KVM_ARM_VCPU_PMU_V3_CTRL:
0976         ret = kvm_arm_pmu_v3_get_attr(vcpu, attr);
0977         break;
0978     case KVM_ARM_VCPU_TIMER_CTRL:
0979         ret = kvm_arm_timer_get_attr(vcpu, attr);
0980         break;
0981     case KVM_ARM_VCPU_PVTIME_CTRL:
0982         ret = kvm_arm_pvtime_get_attr(vcpu, attr);
0983         break;
0984     default:
0985         ret = -ENXIO;
0986         break;
0987     }
0988 
0989     return ret;
0990 }
0991 
0992 int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
0993                    struct kvm_device_attr *attr)
0994 {
0995     int ret;
0996 
0997     switch (attr->group) {
0998     case KVM_ARM_VCPU_PMU_V3_CTRL:
0999         ret = kvm_arm_pmu_v3_has_attr(vcpu, attr);
1000         break;
1001     case KVM_ARM_VCPU_TIMER_CTRL:
1002         ret = kvm_arm_timer_has_attr(vcpu, attr);
1003         break;
1004     case KVM_ARM_VCPU_PVTIME_CTRL:
1005         ret = kvm_arm_pvtime_has_attr(vcpu, attr);
1006         break;
1007     default:
1008         ret = -ENXIO;
1009         break;
1010     }
1011 
1012     return ret;
1013 }
1014 
1015 long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm,
1016                 struct kvm_arm_copy_mte_tags *copy_tags)
1017 {
1018     gpa_t guest_ipa = copy_tags->guest_ipa;
1019     size_t length = copy_tags->length;
1020     void __user *tags = copy_tags->addr;
1021     gpa_t gfn;
1022     bool write = !(copy_tags->flags & KVM_ARM_TAGS_FROM_GUEST);
1023     int ret = 0;
1024 
1025     if (!kvm_has_mte(kvm))
1026         return -EINVAL;
1027 
1028     if (copy_tags->reserved[0] || copy_tags->reserved[1])
1029         return -EINVAL;
1030 
1031     if (copy_tags->flags & ~KVM_ARM_TAGS_FROM_GUEST)
1032         return -EINVAL;
1033 
1034     if (length & ~PAGE_MASK || guest_ipa & ~PAGE_MASK)
1035         return -EINVAL;
1036 
1037     gfn = gpa_to_gfn(guest_ipa);
1038 
1039     mutex_lock(&kvm->slots_lock);
1040 
1041     while (length > 0) {
1042         kvm_pfn_t pfn = gfn_to_pfn_prot(kvm, gfn, write, NULL);
1043         void *maddr;
1044         unsigned long num_tags;
1045         struct page *page;
1046 
1047         if (is_error_noslot_pfn(pfn)) {
1048             ret = -EFAULT;
1049             goto out;
1050         }
1051 
1052         page = pfn_to_online_page(pfn);
1053         if (!page) {
1054             /* Reject ZONE_DEVICE memory */
1055             ret = -EFAULT;
1056             goto out;
1057         }
1058         maddr = page_address(page);
1059 
1060         if (!write) {
1061             if (test_bit(PG_mte_tagged, &page->flags))
1062                 num_tags = mte_copy_tags_to_user(tags, maddr,
1063                             MTE_GRANULES_PER_PAGE);
1064             else
1065                 /* No tags in memory, so write zeros */
1066                 num_tags = MTE_GRANULES_PER_PAGE -
1067                     clear_user(tags, MTE_GRANULES_PER_PAGE);
1068             kvm_release_pfn_clean(pfn);
1069         } else {
1070             num_tags = mte_copy_tags_from_user(maddr, tags,
1071                             MTE_GRANULES_PER_PAGE);
1072 
1073             /*
1074              * Set the flag after checking the write
1075              * completed fully
1076              */
1077             if (num_tags == MTE_GRANULES_PER_PAGE)
1078                 set_bit(PG_mte_tagged, &page->flags);
1079 
1080             kvm_release_pfn_dirty(pfn);
1081         }
1082 
1083         if (num_tags != MTE_GRANULES_PER_PAGE) {
1084             ret = -EFAULT;
1085             goto out;
1086         }
1087 
1088         gfn++;
1089         tags += num_tags;
1090         length -= PAGE_SIZE;
1091     }
1092 
1093 out:
1094     mutex_unlock(&kvm->slots_lock);
1095     /* If some data has been copied report the number of bytes copied */
1096     if (length != copy_tags->length)
1097         return copy_tags->length - length;
1098     return ret;
1099 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!