0001
0002
0003
0004
0005
0006 #include <linux/irqchip/arm-gic.h>
0007 #include <linux/kvm.h>
0008 #include <linux/kvm_host.h>
0009 #include <linux/nospec.h>
0010
0011 #include <kvm/iodev.h>
0012 #include <kvm/arm_vgic.h>
0013
0014 #include "vgic.h"
0015 #include "vgic-mmio.h"
0016
0017
0018
0019
0020
0021
0022
0023
0024
0025 static unsigned long vgic_mmio_read_v2_misc(struct kvm_vcpu *vcpu,
0026 gpa_t addr, unsigned int len)
0027 {
0028 struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
0029 u32 value;
0030
0031 switch (addr & 0x0c) {
0032 case GIC_DIST_CTRL:
0033 value = vgic->enabled ? GICD_ENABLE : 0;
0034 break;
0035 case GIC_DIST_CTR:
0036 value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS;
0037 value = (value >> 5) - 1;
0038 value |= (atomic_read(&vcpu->kvm->online_vcpus) - 1) << 5;
0039 break;
0040 case GIC_DIST_IIDR:
0041 value = (PRODUCT_ID_KVM << GICD_IIDR_PRODUCT_ID_SHIFT) |
0042 (vgic->implementation_rev << GICD_IIDR_REVISION_SHIFT) |
0043 (IMPLEMENTER_ARM << GICD_IIDR_IMPLEMENTER_SHIFT);
0044 break;
0045 default:
0046 return 0;
0047 }
0048
0049 return value;
0050 }
0051
0052 static void vgic_mmio_write_v2_misc(struct kvm_vcpu *vcpu,
0053 gpa_t addr, unsigned int len,
0054 unsigned long val)
0055 {
0056 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
0057 bool was_enabled = dist->enabled;
0058
0059 switch (addr & 0x0c) {
0060 case GIC_DIST_CTRL:
0061 dist->enabled = val & GICD_ENABLE;
0062 if (!was_enabled && dist->enabled)
0063 vgic_kick_vcpus(vcpu->kvm);
0064 break;
0065 case GIC_DIST_CTR:
0066 case GIC_DIST_IIDR:
0067
0068 return;
0069 }
0070 }
0071
0072 static int vgic_mmio_uaccess_write_v2_misc(struct kvm_vcpu *vcpu,
0073 gpa_t addr, unsigned int len,
0074 unsigned long val)
0075 {
0076 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
0077 u32 reg;
0078
0079 switch (addr & 0x0c) {
0080 case GIC_DIST_IIDR:
0081 reg = vgic_mmio_read_v2_misc(vcpu, addr, len);
0082 if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
0083 return -EINVAL;
0084
0085
0086
0087
0088
0089
0090
0091
0092
0093
0094 reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
0095 switch (reg) {
0096 case KVM_VGIC_IMP_REV_2:
0097 case KVM_VGIC_IMP_REV_3:
0098 vcpu->kvm->arch.vgic.v2_groups_user_writable = true;
0099 dist->implementation_rev = reg;
0100 return 0;
0101 default:
0102 return -EINVAL;
0103 }
0104 }
0105
0106 vgic_mmio_write_v2_misc(vcpu, addr, len, val);
0107 return 0;
0108 }
0109
0110 static int vgic_mmio_uaccess_write_v2_group(struct kvm_vcpu *vcpu,
0111 gpa_t addr, unsigned int len,
0112 unsigned long val)
0113 {
0114 if (vcpu->kvm->arch.vgic.v2_groups_user_writable)
0115 vgic_mmio_write_group(vcpu, addr, len, val);
0116
0117 return 0;
0118 }
0119
0120 static void vgic_mmio_write_sgir(struct kvm_vcpu *source_vcpu,
0121 gpa_t addr, unsigned int len,
0122 unsigned long val)
0123 {
0124 int nr_vcpus = atomic_read(&source_vcpu->kvm->online_vcpus);
0125 int intid = val & 0xf;
0126 int targets = (val >> 16) & 0xff;
0127 int mode = (val >> 24) & 0x03;
0128 struct kvm_vcpu *vcpu;
0129 unsigned long flags, c;
0130
0131 switch (mode) {
0132 case 0x0:
0133 break;
0134 case 0x1:
0135 targets = (1U << nr_vcpus) - 1;
0136 targets &= ~(1U << source_vcpu->vcpu_id);
0137 break;
0138 case 0x2:
0139 targets = (1U << source_vcpu->vcpu_id);
0140 break;
0141 case 0x3:
0142 return;
0143 }
0144
0145 kvm_for_each_vcpu(c, vcpu, source_vcpu->kvm) {
0146 struct vgic_irq *irq;
0147
0148 if (!(targets & (1U << c)))
0149 continue;
0150
0151 irq = vgic_get_irq(source_vcpu->kvm, vcpu, intid);
0152
0153 raw_spin_lock_irqsave(&irq->irq_lock, flags);
0154 irq->pending_latch = true;
0155 irq->source |= 1U << source_vcpu->vcpu_id;
0156
0157 vgic_queue_irq_unlock(source_vcpu->kvm, irq, flags);
0158 vgic_put_irq(source_vcpu->kvm, irq);
0159 }
0160 }
0161
0162 static unsigned long vgic_mmio_read_target(struct kvm_vcpu *vcpu,
0163 gpa_t addr, unsigned int len)
0164 {
0165 u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
0166 int i;
0167 u64 val = 0;
0168
0169 for (i = 0; i < len; i++) {
0170 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
0171
0172 val |= (u64)irq->targets << (i * 8);
0173
0174 vgic_put_irq(vcpu->kvm, irq);
0175 }
0176
0177 return val;
0178 }
0179
0180 static void vgic_mmio_write_target(struct kvm_vcpu *vcpu,
0181 gpa_t addr, unsigned int len,
0182 unsigned long val)
0183 {
0184 u32 intid = VGIC_ADDR_TO_INTID(addr, 8);
0185 u8 cpu_mask = GENMASK(atomic_read(&vcpu->kvm->online_vcpus) - 1, 0);
0186 int i;
0187 unsigned long flags;
0188
0189
0190 if (intid < VGIC_NR_PRIVATE_IRQS)
0191 return;
0192
0193 for (i = 0; i < len; i++) {
0194 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid + i);
0195 int target;
0196
0197 raw_spin_lock_irqsave(&irq->irq_lock, flags);
0198
0199 irq->targets = (val >> (i * 8)) & cpu_mask;
0200 target = irq->targets ? __ffs(irq->targets) : 0;
0201 irq->target_vcpu = kvm_get_vcpu(vcpu->kvm, target);
0202
0203 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
0204 vgic_put_irq(vcpu->kvm, irq);
0205 }
0206 }
0207
0208 static unsigned long vgic_mmio_read_sgipend(struct kvm_vcpu *vcpu,
0209 gpa_t addr, unsigned int len)
0210 {
0211 u32 intid = addr & 0x0f;
0212 int i;
0213 u64 val = 0;
0214
0215 for (i = 0; i < len; i++) {
0216 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
0217
0218 val |= (u64)irq->source << (i * 8);
0219
0220 vgic_put_irq(vcpu->kvm, irq);
0221 }
0222 return val;
0223 }
0224
0225 static void vgic_mmio_write_sgipendc(struct kvm_vcpu *vcpu,
0226 gpa_t addr, unsigned int len,
0227 unsigned long val)
0228 {
0229 u32 intid = addr & 0x0f;
0230 int i;
0231 unsigned long flags;
0232
0233 for (i = 0; i < len; i++) {
0234 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
0235
0236 raw_spin_lock_irqsave(&irq->irq_lock, flags);
0237
0238 irq->source &= ~((val >> (i * 8)) & 0xff);
0239 if (!irq->source)
0240 irq->pending_latch = false;
0241
0242 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
0243 vgic_put_irq(vcpu->kvm, irq);
0244 }
0245 }
0246
0247 static void vgic_mmio_write_sgipends(struct kvm_vcpu *vcpu,
0248 gpa_t addr, unsigned int len,
0249 unsigned long val)
0250 {
0251 u32 intid = addr & 0x0f;
0252 int i;
0253 unsigned long flags;
0254
0255 for (i = 0; i < len; i++) {
0256 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
0257
0258 raw_spin_lock_irqsave(&irq->irq_lock, flags);
0259
0260 irq->source |= (val >> (i * 8)) & 0xff;
0261
0262 if (irq->source) {
0263 irq->pending_latch = true;
0264 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
0265 } else {
0266 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
0267 }
0268 vgic_put_irq(vcpu->kvm, irq);
0269 }
0270 }
0271
0272 #define GICC_ARCH_VERSION_V2 0x2
0273
0274
0275 static unsigned long vgic_mmio_read_vcpuif(struct kvm_vcpu *vcpu,
0276 gpa_t addr, unsigned int len)
0277 {
0278 struct vgic_vmcr vmcr;
0279 u32 val;
0280
0281 vgic_get_vmcr(vcpu, &vmcr);
0282
0283 switch (addr & 0xff) {
0284 case GIC_CPU_CTRL:
0285 val = vmcr.grpen0 << GIC_CPU_CTRL_EnableGrp0_SHIFT;
0286 val |= vmcr.grpen1 << GIC_CPU_CTRL_EnableGrp1_SHIFT;
0287 val |= vmcr.ackctl << GIC_CPU_CTRL_AckCtl_SHIFT;
0288 val |= vmcr.fiqen << GIC_CPU_CTRL_FIQEn_SHIFT;
0289 val |= vmcr.cbpr << GIC_CPU_CTRL_CBPR_SHIFT;
0290 val |= vmcr.eoim << GIC_CPU_CTRL_EOImodeNS_SHIFT;
0291
0292 break;
0293 case GIC_CPU_PRIMASK:
0294
0295
0296
0297
0298
0299
0300
0301 val = (vmcr.pmr & GICV_PMR_PRIORITY_MASK) >>
0302 GICV_PMR_PRIORITY_SHIFT;
0303 break;
0304 case GIC_CPU_BINPOINT:
0305 val = vmcr.bpr;
0306 break;
0307 case GIC_CPU_ALIAS_BINPOINT:
0308 val = vmcr.abpr;
0309 break;
0310 case GIC_CPU_IDENT:
0311 val = ((PRODUCT_ID_KVM << 20) |
0312 (GICC_ARCH_VERSION_V2 << 16) |
0313 IMPLEMENTER_ARM);
0314 break;
0315 default:
0316 return 0;
0317 }
0318
0319 return val;
0320 }
0321
0322 static void vgic_mmio_write_vcpuif(struct kvm_vcpu *vcpu,
0323 gpa_t addr, unsigned int len,
0324 unsigned long val)
0325 {
0326 struct vgic_vmcr vmcr;
0327
0328 vgic_get_vmcr(vcpu, &vmcr);
0329
0330 switch (addr & 0xff) {
0331 case GIC_CPU_CTRL:
0332 vmcr.grpen0 = !!(val & GIC_CPU_CTRL_EnableGrp0);
0333 vmcr.grpen1 = !!(val & GIC_CPU_CTRL_EnableGrp1);
0334 vmcr.ackctl = !!(val & GIC_CPU_CTRL_AckCtl);
0335 vmcr.fiqen = !!(val & GIC_CPU_CTRL_FIQEn);
0336 vmcr.cbpr = !!(val & GIC_CPU_CTRL_CBPR);
0337 vmcr.eoim = !!(val & GIC_CPU_CTRL_EOImodeNS);
0338
0339 break;
0340 case GIC_CPU_PRIMASK:
0341
0342
0343
0344
0345
0346
0347
0348 vmcr.pmr = (val << GICV_PMR_PRIORITY_SHIFT) &
0349 GICV_PMR_PRIORITY_MASK;
0350 break;
0351 case GIC_CPU_BINPOINT:
0352 vmcr.bpr = val;
0353 break;
0354 case GIC_CPU_ALIAS_BINPOINT:
0355 vmcr.abpr = val;
0356 break;
0357 }
0358
0359 vgic_set_vmcr(vcpu, &vmcr);
0360 }
0361
0362 static unsigned long vgic_mmio_read_apr(struct kvm_vcpu *vcpu,
0363 gpa_t addr, unsigned int len)
0364 {
0365 int n;
0366
0367 n = (addr >> 2) & 0x3;
0368
0369 if (kvm_vgic_global_state.type == VGIC_V2) {
0370
0371 if (n != 0)
0372 return 0;
0373 return vcpu->arch.vgic_cpu.vgic_v2.vgic_apr;
0374 } else {
0375 struct vgic_v3_cpu_if *vgicv3 = &vcpu->arch.vgic_cpu.vgic_v3;
0376
0377 if (n > vgic_v3_max_apr_idx(vcpu))
0378 return 0;
0379
0380 n = array_index_nospec(n, 4);
0381
0382
0383 return vgicv3->vgic_ap1r[n];
0384 }
0385 }
0386
0387 static void vgic_mmio_write_apr(struct kvm_vcpu *vcpu,
0388 gpa_t addr, unsigned int len,
0389 unsigned long val)
0390 {
0391 int n;
0392
0393 n = (addr >> 2) & 0x3;
0394
0395 if (kvm_vgic_global_state.type == VGIC_V2) {
0396
0397 if (n != 0)
0398 return;
0399 vcpu->arch.vgic_cpu.vgic_v2.vgic_apr = val;
0400 } else {
0401 struct vgic_v3_cpu_if *vgicv3 = &vcpu->arch.vgic_cpu.vgic_v3;
0402
0403 if (n > vgic_v3_max_apr_idx(vcpu))
0404 return;
0405
0406 n = array_index_nospec(n, 4);
0407
0408
0409 vgicv3->vgic_ap1r[n] = val;
0410 }
0411 }
0412
0413 static const struct vgic_register_region vgic_v2_dist_registers[] = {
0414 REGISTER_DESC_WITH_LENGTH_UACCESS(GIC_DIST_CTRL,
0415 vgic_mmio_read_v2_misc, vgic_mmio_write_v2_misc,
0416 NULL, vgic_mmio_uaccess_write_v2_misc,
0417 12, VGIC_ACCESS_32bit),
0418 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_IGROUP,
0419 vgic_mmio_read_group, vgic_mmio_write_group,
0420 NULL, vgic_mmio_uaccess_write_v2_group, 1,
0421 VGIC_ACCESS_32bit),
0422 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_ENABLE_SET,
0423 vgic_mmio_read_enable, vgic_mmio_write_senable,
0424 NULL, vgic_uaccess_write_senable, 1,
0425 VGIC_ACCESS_32bit),
0426 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_ENABLE_CLEAR,
0427 vgic_mmio_read_enable, vgic_mmio_write_cenable,
0428 NULL, vgic_uaccess_write_cenable, 1,
0429 VGIC_ACCESS_32bit),
0430 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_PENDING_SET,
0431 vgic_mmio_read_pending, vgic_mmio_write_spending,
0432 vgic_uaccess_read_pending, vgic_uaccess_write_spending, 1,
0433 VGIC_ACCESS_32bit),
0434 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_PENDING_CLEAR,
0435 vgic_mmio_read_pending, vgic_mmio_write_cpending,
0436 vgic_uaccess_read_pending, vgic_uaccess_write_cpending, 1,
0437 VGIC_ACCESS_32bit),
0438 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_ACTIVE_SET,
0439 vgic_mmio_read_active, vgic_mmio_write_sactive,
0440 vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 1,
0441 VGIC_ACCESS_32bit),
0442 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_ACTIVE_CLEAR,
0443 vgic_mmio_read_active, vgic_mmio_write_cactive,
0444 vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive, 1,
0445 VGIC_ACCESS_32bit),
0446 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_PRI,
0447 vgic_mmio_read_priority, vgic_mmio_write_priority, NULL, NULL,
0448 8, VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
0449 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_TARGET,
0450 vgic_mmio_read_target, vgic_mmio_write_target, NULL, NULL, 8,
0451 VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
0452 REGISTER_DESC_WITH_BITS_PER_IRQ(GIC_DIST_CONFIG,
0453 vgic_mmio_read_config, vgic_mmio_write_config, NULL, NULL, 2,
0454 VGIC_ACCESS_32bit),
0455 REGISTER_DESC_WITH_LENGTH(GIC_DIST_SOFTINT,
0456 vgic_mmio_read_raz, vgic_mmio_write_sgir, 4,
0457 VGIC_ACCESS_32bit),
0458 REGISTER_DESC_WITH_LENGTH(GIC_DIST_SGI_PENDING_CLEAR,
0459 vgic_mmio_read_sgipend, vgic_mmio_write_sgipendc, 16,
0460 VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
0461 REGISTER_DESC_WITH_LENGTH(GIC_DIST_SGI_PENDING_SET,
0462 vgic_mmio_read_sgipend, vgic_mmio_write_sgipends, 16,
0463 VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
0464 };
0465
0466 static const struct vgic_register_region vgic_v2_cpu_registers[] = {
0467 REGISTER_DESC_WITH_LENGTH(GIC_CPU_CTRL,
0468 vgic_mmio_read_vcpuif, vgic_mmio_write_vcpuif, 4,
0469 VGIC_ACCESS_32bit),
0470 REGISTER_DESC_WITH_LENGTH(GIC_CPU_PRIMASK,
0471 vgic_mmio_read_vcpuif, vgic_mmio_write_vcpuif, 4,
0472 VGIC_ACCESS_32bit),
0473 REGISTER_DESC_WITH_LENGTH(GIC_CPU_BINPOINT,
0474 vgic_mmio_read_vcpuif, vgic_mmio_write_vcpuif, 4,
0475 VGIC_ACCESS_32bit),
0476 REGISTER_DESC_WITH_LENGTH(GIC_CPU_ALIAS_BINPOINT,
0477 vgic_mmio_read_vcpuif, vgic_mmio_write_vcpuif, 4,
0478 VGIC_ACCESS_32bit),
0479 REGISTER_DESC_WITH_LENGTH(GIC_CPU_ACTIVEPRIO,
0480 vgic_mmio_read_apr, vgic_mmio_write_apr, 16,
0481 VGIC_ACCESS_32bit),
0482 REGISTER_DESC_WITH_LENGTH(GIC_CPU_IDENT,
0483 vgic_mmio_read_vcpuif, vgic_mmio_write_vcpuif, 4,
0484 VGIC_ACCESS_32bit),
0485 };
0486
0487 unsigned int vgic_v2_init_dist_iodev(struct vgic_io_device *dev)
0488 {
0489 dev->regions = vgic_v2_dist_registers;
0490 dev->nr_regions = ARRAY_SIZE(vgic_v2_dist_registers);
0491
0492 kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops);
0493
0494 return SZ_4K;
0495 }
0496
0497 int vgic_v2_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr)
0498 {
0499 const struct vgic_register_region *region;
0500 struct vgic_io_device iodev;
0501 struct vgic_reg_attr reg_attr;
0502 struct kvm_vcpu *vcpu;
0503 gpa_t addr;
0504 int ret;
0505
0506 ret = vgic_v2_parse_attr(dev, attr, ®_attr);
0507 if (ret)
0508 return ret;
0509
0510 vcpu = reg_attr.vcpu;
0511 addr = reg_attr.addr;
0512
0513 switch (attr->group) {
0514 case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
0515 iodev.regions = vgic_v2_dist_registers;
0516 iodev.nr_regions = ARRAY_SIZE(vgic_v2_dist_registers);
0517 iodev.base_addr = 0;
0518 break;
0519 case KVM_DEV_ARM_VGIC_GRP_CPU_REGS:
0520 iodev.regions = vgic_v2_cpu_registers;
0521 iodev.nr_regions = ARRAY_SIZE(vgic_v2_cpu_registers);
0522 iodev.base_addr = 0;
0523 break;
0524 default:
0525 return -ENXIO;
0526 }
0527
0528
0529 if (addr & 3)
0530 return -ENXIO;
0531
0532 region = vgic_get_mmio_region(vcpu, &iodev, addr, sizeof(u32));
0533 if (!region)
0534 return -ENXIO;
0535
0536 return 0;
0537 }
0538
0539 int vgic_v2_cpuif_uaccess(struct kvm_vcpu *vcpu, bool is_write,
0540 int offset, u32 *val)
0541 {
0542 struct vgic_io_device dev = {
0543 .regions = vgic_v2_cpu_registers,
0544 .nr_regions = ARRAY_SIZE(vgic_v2_cpu_registers),
0545 .iodev_type = IODEV_CPUIF,
0546 };
0547
0548 return vgic_uaccess(vcpu, &dev, is_write, offset, val);
0549 }
0550
0551 int vgic_v2_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
0552 int offset, u32 *val)
0553 {
0554 struct vgic_io_device dev = {
0555 .regions = vgic_v2_dist_registers,
0556 .nr_regions = ARRAY_SIZE(vgic_v2_dist_registers),
0557 .iodev_type = IODEV_DIST,
0558 };
0559
0560 return vgic_uaccess(vcpu, &dev, is_write, offset, val);
0561 }