Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * VGICv3 MMIO handling functions
0004  */
0005 
0006 #include <linux/bitfield.h>
0007 #include <linux/irqchip/arm-gic-v3.h>
0008 #include <linux/kvm.h>
0009 #include <linux/kvm_host.h>
0010 #include <linux/interrupt.h>
0011 #include <kvm/iodev.h>
0012 #include <kvm/arm_vgic.h>
0013 
0014 #include <asm/kvm_emulate.h>
0015 #include <asm/kvm_arm.h>
0016 #include <asm/kvm_mmu.h>
0017 
0018 #include "vgic.h"
0019 #include "vgic-mmio.h"
0020 
0021 /* extract @num bytes at @offset bytes offset in data */
0022 unsigned long extract_bytes(u64 data, unsigned int offset,
0023                 unsigned int num)
0024 {
0025     return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0);
0026 }
0027 
0028 /* allows updates of any half of a 64-bit register (or the whole thing) */
0029 u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len,
0030              unsigned long val)
0031 {
0032     int lower = (offset & 4) * 8;
0033     int upper = lower + 8 * len - 1;
0034 
0035     reg &= ~GENMASK_ULL(upper, lower);
0036     val &= GENMASK_ULL(len * 8 - 1, 0);
0037 
0038     return reg | ((u64)val << lower);
0039 }
0040 
0041 bool vgic_has_its(struct kvm *kvm)
0042 {
0043     struct vgic_dist *dist = &kvm->arch.vgic;
0044 
0045     if (dist->vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
0046         return false;
0047 
0048     return dist->has_its;
0049 }
0050 
0051 bool vgic_supports_direct_msis(struct kvm *kvm)
0052 {
0053     return (kvm_vgic_global_state.has_gicv4_1 ||
0054         (kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm)));
0055 }
0056 
0057 /*
0058  * The Revision field in the IIDR have the following meanings:
0059  *
0060  * Revision 2: Interrupt groups are guest-configurable and signaled using
0061  *         their configured groups.
0062  */
0063 
0064 static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
0065                         gpa_t addr, unsigned int len)
0066 {
0067     struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
0068     u32 value = 0;
0069 
0070     switch (addr & 0x0c) {
0071     case GICD_CTLR:
0072         if (vgic->enabled)
0073             value |= GICD_CTLR_ENABLE_SS_G1;
0074         value |= GICD_CTLR_ARE_NS | GICD_CTLR_DS;
0075         if (vgic->nassgireq)
0076             value |= GICD_CTLR_nASSGIreq;
0077         break;
0078     case GICD_TYPER:
0079         value = vgic->nr_spis + VGIC_NR_PRIVATE_IRQS;
0080         value = (value >> 5) - 1;
0081         if (vgic_has_its(vcpu->kvm)) {
0082             value |= (INTERRUPT_ID_BITS_ITS - 1) << 19;
0083             value |= GICD_TYPER_LPIS;
0084         } else {
0085             value |= (INTERRUPT_ID_BITS_SPIS - 1) << 19;
0086         }
0087         break;
0088     case GICD_TYPER2:
0089         if (kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi())
0090             value = GICD_TYPER2_nASSGIcap;
0091         break;
0092     case GICD_IIDR:
0093         value = (PRODUCT_ID_KVM << GICD_IIDR_PRODUCT_ID_SHIFT) |
0094             (vgic->implementation_rev << GICD_IIDR_REVISION_SHIFT) |
0095             (IMPLEMENTER_ARM << GICD_IIDR_IMPLEMENTER_SHIFT);
0096         break;
0097     default:
0098         return 0;
0099     }
0100 
0101     return value;
0102 }
0103 
0104 static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu,
0105                     gpa_t addr, unsigned int len,
0106                     unsigned long val)
0107 {
0108     struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
0109 
0110     switch (addr & 0x0c) {
0111     case GICD_CTLR: {
0112         bool was_enabled, is_hwsgi;
0113 
0114         mutex_lock(&vcpu->kvm->lock);
0115 
0116         was_enabled = dist->enabled;
0117         is_hwsgi = dist->nassgireq;
0118 
0119         dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
0120 
0121         /* Not a GICv4.1? No HW SGIs */
0122         if (!kvm_vgic_global_state.has_gicv4_1 || !gic_cpuif_has_vsgi())
0123             val &= ~GICD_CTLR_nASSGIreq;
0124 
0125         /* Dist stays enabled? nASSGIreq is RO */
0126         if (was_enabled && dist->enabled) {
0127             val &= ~GICD_CTLR_nASSGIreq;
0128             val |= FIELD_PREP(GICD_CTLR_nASSGIreq, is_hwsgi);
0129         }
0130 
0131         /* Switching HW SGIs? */
0132         dist->nassgireq = val & GICD_CTLR_nASSGIreq;
0133         if (is_hwsgi != dist->nassgireq)
0134             vgic_v4_configure_vsgis(vcpu->kvm);
0135 
0136         if (kvm_vgic_global_state.has_gicv4_1 &&
0137             was_enabled != dist->enabled)
0138             kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4);
0139         else if (!was_enabled && dist->enabled)
0140             vgic_kick_vcpus(vcpu->kvm);
0141 
0142         mutex_unlock(&vcpu->kvm->lock);
0143         break;
0144     }
0145     case GICD_TYPER:
0146     case GICD_TYPER2:
0147     case GICD_IIDR:
0148         /* This is at best for documentation purposes... */
0149         return;
0150     }
0151 }
0152 
0153 static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
0154                        gpa_t addr, unsigned int len,
0155                        unsigned long val)
0156 {
0157     struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
0158     u32 reg;
0159 
0160     switch (addr & 0x0c) {
0161     case GICD_TYPER2:
0162         if (val != vgic_mmio_read_v3_misc(vcpu, addr, len))
0163             return -EINVAL;
0164         return 0;
0165     case GICD_IIDR:
0166         reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
0167         if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
0168             return -EINVAL;
0169 
0170         reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
0171         switch (reg) {
0172         case KVM_VGIC_IMP_REV_2:
0173         case KVM_VGIC_IMP_REV_3:
0174             dist->implementation_rev = reg;
0175             return 0;
0176         default:
0177             return -EINVAL;
0178         }
0179     case GICD_CTLR:
0180         /* Not a GICv4.1? No HW SGIs */
0181         if (!kvm_vgic_global_state.has_gicv4_1)
0182             val &= ~GICD_CTLR_nASSGIreq;
0183 
0184         dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
0185         dist->nassgireq = val & GICD_CTLR_nASSGIreq;
0186         return 0;
0187     }
0188 
0189     vgic_mmio_write_v3_misc(vcpu, addr, len, val);
0190     return 0;
0191 }
0192 
0193 static unsigned long vgic_mmio_read_irouter(struct kvm_vcpu *vcpu,
0194                         gpa_t addr, unsigned int len)
0195 {
0196     int intid = VGIC_ADDR_TO_INTID(addr, 64);
0197     struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid);
0198     unsigned long ret = 0;
0199 
0200     if (!irq)
0201         return 0;
0202 
0203     /* The upper word is RAZ for us. */
0204     if (!(addr & 4))
0205         ret = extract_bytes(READ_ONCE(irq->mpidr), addr & 7, len);
0206 
0207     vgic_put_irq(vcpu->kvm, irq);
0208     return ret;
0209 }
0210 
0211 static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu,
0212                     gpa_t addr, unsigned int len,
0213                     unsigned long val)
0214 {
0215     int intid = VGIC_ADDR_TO_INTID(addr, 64);
0216     struct vgic_irq *irq;
0217     unsigned long flags;
0218 
0219     /* The upper word is WI for us since we don't implement Aff3. */
0220     if (addr & 4)
0221         return;
0222 
0223     irq = vgic_get_irq(vcpu->kvm, NULL, intid);
0224 
0225     if (!irq)
0226         return;
0227 
0228     raw_spin_lock_irqsave(&irq->irq_lock, flags);
0229 
0230     /* We only care about and preserve Aff0, Aff1 and Aff2. */
0231     irq->mpidr = val & GENMASK(23, 0);
0232     irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr);
0233 
0234     raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
0235     vgic_put_irq(vcpu->kvm, irq);
0236 }
0237 
0238 bool vgic_lpis_enabled(struct kvm_vcpu *vcpu)
0239 {
0240     struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
0241 
0242     return atomic_read(&vgic_cpu->ctlr) == GICR_CTLR_ENABLE_LPIS;
0243 }
0244 
0245 static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu,
0246                          gpa_t addr, unsigned int len)
0247 {
0248     struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
0249     unsigned long val;
0250 
0251     val = atomic_read(&vgic_cpu->ctlr);
0252     if (vgic_get_implementation_rev(vcpu) >= KVM_VGIC_IMP_REV_3)
0253         val |= GICR_CTLR_IR | GICR_CTLR_CES;
0254 
0255     return val;
0256 }
0257 
0258 static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
0259                      gpa_t addr, unsigned int len,
0260                      unsigned long val)
0261 {
0262     struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
0263     u32 ctlr;
0264 
0265     if (!vgic_has_its(vcpu->kvm))
0266         return;
0267 
0268     if (!(val & GICR_CTLR_ENABLE_LPIS)) {
0269         /*
0270          * Don't disable if RWP is set, as there already an
0271          * ongoing disable. Funky guest...
0272          */
0273         ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr,
0274                           GICR_CTLR_ENABLE_LPIS,
0275                           GICR_CTLR_RWP);
0276         if (ctlr != GICR_CTLR_ENABLE_LPIS)
0277             return;
0278 
0279         vgic_flush_pending_lpis(vcpu);
0280         vgic_its_invalidate_cache(vcpu->kvm);
0281         atomic_set_release(&vgic_cpu->ctlr, 0);
0282     } else {
0283         ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0,
0284                           GICR_CTLR_ENABLE_LPIS);
0285         if (ctlr != 0)
0286             return;
0287 
0288         vgic_enable_lpis(vcpu);
0289     }
0290 }
0291 
0292 static bool vgic_mmio_vcpu_rdist_is_last(struct kvm_vcpu *vcpu)
0293 {
0294     struct vgic_dist *vgic = &vcpu->kvm->arch.vgic;
0295     struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
0296     struct vgic_redist_region *iter, *rdreg = vgic_cpu->rdreg;
0297 
0298     if (!rdreg)
0299         return false;
0300 
0301     if (vgic_cpu->rdreg_index < rdreg->free_index - 1) {
0302         return false;
0303     } else if (rdreg->count && vgic_cpu->rdreg_index == (rdreg->count - 1)) {
0304         struct list_head *rd_regions = &vgic->rd_regions;
0305         gpa_t end = rdreg->base + rdreg->count * KVM_VGIC_V3_REDIST_SIZE;
0306 
0307         /*
0308          * the rdist is the last one of the redist region,
0309          * check whether there is no other contiguous rdist region
0310          */
0311         list_for_each_entry(iter, rd_regions, list) {
0312             if (iter->base == end && iter->free_index > 0)
0313                 return false;
0314         }
0315     }
0316     return true;
0317 }
0318 
0319 static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu,
0320                           gpa_t addr, unsigned int len)
0321 {
0322     unsigned long mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
0323     int target_vcpu_id = vcpu->vcpu_id;
0324     u64 value;
0325 
0326     value = (u64)(mpidr & GENMASK(23, 0)) << 32;
0327     value |= ((target_vcpu_id & 0xffff) << 8);
0328 
0329     if (vgic_has_its(vcpu->kvm))
0330         value |= GICR_TYPER_PLPIS;
0331 
0332     if (vgic_mmio_vcpu_rdist_is_last(vcpu))
0333         value |= GICR_TYPER_LAST;
0334 
0335     return extract_bytes(value, addr & 7, len);
0336 }
0337 
0338 static unsigned long vgic_mmio_read_v3r_iidr(struct kvm_vcpu *vcpu,
0339                          gpa_t addr, unsigned int len)
0340 {
0341     return (PRODUCT_ID_KVM << 24) | (IMPLEMENTER_ARM << 0);
0342 }
0343 
0344 static unsigned long vgic_mmio_read_v3_idregs(struct kvm_vcpu *vcpu,
0345                           gpa_t addr, unsigned int len)
0346 {
0347     switch (addr & 0xffff) {
0348     case GICD_PIDR2:
0349         /* report a GICv3 compliant implementation */
0350         return 0x3b;
0351     }
0352 
0353     return 0;
0354 }
0355 
0356 static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu,
0357                      gpa_t addr, unsigned int len,
0358                      unsigned long val)
0359 {
0360     u32 intid = VGIC_ADDR_TO_INTID(addr, 1);
0361     int i;
0362     unsigned long flags;
0363 
0364     for (i = 0; i < len * 8; i++) {
0365         struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
0366 
0367         raw_spin_lock_irqsave(&irq->irq_lock, flags);
0368         if (test_bit(i, &val)) {
0369             /*
0370              * pending_latch is set irrespective of irq type
0371              * (level or edge) to avoid dependency that VM should
0372              * restore irq config before pending info.
0373              */
0374             irq->pending_latch = true;
0375             vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
0376         } else {
0377             irq->pending_latch = false;
0378             raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
0379         }
0380 
0381         vgic_put_irq(vcpu->kvm, irq);
0382     }
0383 
0384     return 0;
0385 }
0386 
0387 /* We want to avoid outer shareable. */
0388 u64 vgic_sanitise_shareability(u64 field)
0389 {
0390     switch (field) {
0391     case GIC_BASER_OuterShareable:
0392         return GIC_BASER_InnerShareable;
0393     default:
0394         return field;
0395     }
0396 }
0397 
0398 /* Avoid any inner non-cacheable mapping. */
0399 u64 vgic_sanitise_inner_cacheability(u64 field)
0400 {
0401     switch (field) {
0402     case GIC_BASER_CACHE_nCnB:
0403     case GIC_BASER_CACHE_nC:
0404         return GIC_BASER_CACHE_RaWb;
0405     default:
0406         return field;
0407     }
0408 }
0409 
0410 /* Non-cacheable or same-as-inner are OK. */
0411 u64 vgic_sanitise_outer_cacheability(u64 field)
0412 {
0413     switch (field) {
0414     case GIC_BASER_CACHE_SameAsInner:
0415     case GIC_BASER_CACHE_nC:
0416         return field;
0417     default:
0418         return GIC_BASER_CACHE_SameAsInner;
0419     }
0420 }
0421 
0422 u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift,
0423             u64 (*sanitise_fn)(u64))
0424 {
0425     u64 field = (reg & field_mask) >> field_shift;
0426 
0427     field = sanitise_fn(field) << field_shift;
0428     return (reg & ~field_mask) | field;
0429 }
0430 
0431 #define PROPBASER_RES0_MASK                     \
0432     (GENMASK_ULL(63, 59) | GENMASK_ULL(55, 52) | GENMASK_ULL(6, 5))
0433 #define PENDBASER_RES0_MASK                     \
0434     (BIT_ULL(63) | GENMASK_ULL(61, 59) | GENMASK_ULL(55, 52) |  \
0435      GENMASK_ULL(15, 12) | GENMASK_ULL(6, 0))
0436 
0437 static u64 vgic_sanitise_pendbaser(u64 reg)
0438 {
0439     reg = vgic_sanitise_field(reg, GICR_PENDBASER_SHAREABILITY_MASK,
0440                   GICR_PENDBASER_SHAREABILITY_SHIFT,
0441                   vgic_sanitise_shareability);
0442     reg = vgic_sanitise_field(reg, GICR_PENDBASER_INNER_CACHEABILITY_MASK,
0443                   GICR_PENDBASER_INNER_CACHEABILITY_SHIFT,
0444                   vgic_sanitise_inner_cacheability);
0445     reg = vgic_sanitise_field(reg, GICR_PENDBASER_OUTER_CACHEABILITY_MASK,
0446                   GICR_PENDBASER_OUTER_CACHEABILITY_SHIFT,
0447                   vgic_sanitise_outer_cacheability);
0448 
0449     reg &= ~PENDBASER_RES0_MASK;
0450 
0451     return reg;
0452 }
0453 
0454 static u64 vgic_sanitise_propbaser(u64 reg)
0455 {
0456     reg = vgic_sanitise_field(reg, GICR_PROPBASER_SHAREABILITY_MASK,
0457                   GICR_PROPBASER_SHAREABILITY_SHIFT,
0458                   vgic_sanitise_shareability);
0459     reg = vgic_sanitise_field(reg, GICR_PROPBASER_INNER_CACHEABILITY_MASK,
0460                   GICR_PROPBASER_INNER_CACHEABILITY_SHIFT,
0461                   vgic_sanitise_inner_cacheability);
0462     reg = vgic_sanitise_field(reg, GICR_PROPBASER_OUTER_CACHEABILITY_MASK,
0463                   GICR_PROPBASER_OUTER_CACHEABILITY_SHIFT,
0464                   vgic_sanitise_outer_cacheability);
0465 
0466     reg &= ~PROPBASER_RES0_MASK;
0467     return reg;
0468 }
0469 
0470 static unsigned long vgic_mmio_read_propbase(struct kvm_vcpu *vcpu,
0471                          gpa_t addr, unsigned int len)
0472 {
0473     struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
0474 
0475     return extract_bytes(dist->propbaser, addr & 7, len);
0476 }
0477 
0478 static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu,
0479                      gpa_t addr, unsigned int len,
0480                      unsigned long val)
0481 {
0482     struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
0483     u64 old_propbaser, propbaser;
0484 
0485     /* Storing a value with LPIs already enabled is undefined */
0486     if (vgic_lpis_enabled(vcpu))
0487         return;
0488 
0489     do {
0490         old_propbaser = READ_ONCE(dist->propbaser);
0491         propbaser = old_propbaser;
0492         propbaser = update_64bit_reg(propbaser, addr & 4, len, val);
0493         propbaser = vgic_sanitise_propbaser(propbaser);
0494     } while (cmpxchg64(&dist->propbaser, old_propbaser,
0495                propbaser) != old_propbaser);
0496 }
0497 
0498 static unsigned long vgic_mmio_read_pendbase(struct kvm_vcpu *vcpu,
0499                          gpa_t addr, unsigned int len)
0500 {
0501     struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
0502     u64 value = vgic_cpu->pendbaser;
0503 
0504     value &= ~GICR_PENDBASER_PTZ;
0505 
0506     return extract_bytes(value, addr & 7, len);
0507 }
0508 
0509 static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu,
0510                      gpa_t addr, unsigned int len,
0511                      unsigned long val)
0512 {
0513     struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
0514     u64 old_pendbaser, pendbaser;
0515 
0516     /* Storing a value with LPIs already enabled is undefined */
0517     if (vgic_lpis_enabled(vcpu))
0518         return;
0519 
0520     do {
0521         old_pendbaser = READ_ONCE(vgic_cpu->pendbaser);
0522         pendbaser = old_pendbaser;
0523         pendbaser = update_64bit_reg(pendbaser, addr & 4, len, val);
0524         pendbaser = vgic_sanitise_pendbaser(pendbaser);
0525     } while (cmpxchg64(&vgic_cpu->pendbaser, old_pendbaser,
0526                pendbaser) != old_pendbaser);
0527 }
0528 
0529 static unsigned long vgic_mmio_read_sync(struct kvm_vcpu *vcpu,
0530                      gpa_t addr, unsigned int len)
0531 {
0532     return !!atomic_read(&vcpu->arch.vgic_cpu.syncr_busy);
0533 }
0534 
0535 static void vgic_set_rdist_busy(struct kvm_vcpu *vcpu, bool busy)
0536 {
0537     if (busy) {
0538         atomic_inc(&vcpu->arch.vgic_cpu.syncr_busy);
0539         smp_mb__after_atomic();
0540     } else {
0541         smp_mb__before_atomic();
0542         atomic_dec(&vcpu->arch.vgic_cpu.syncr_busy);
0543     }
0544 }
0545 
0546 static void vgic_mmio_write_invlpi(struct kvm_vcpu *vcpu,
0547                    gpa_t addr, unsigned int len,
0548                    unsigned long val)
0549 {
0550     struct vgic_irq *irq;
0551 
0552     /*
0553      * If the guest wrote only to the upper 32bit part of the
0554      * register, drop the write on the floor, as it is only for
0555      * vPEs (which we don't support for obvious reasons).
0556      *
0557      * Also discard the access if LPIs are not enabled.
0558      */
0559     if ((addr & 4) || !vgic_lpis_enabled(vcpu))
0560         return;
0561 
0562     vgic_set_rdist_busy(vcpu, true);
0563 
0564     irq = vgic_get_irq(vcpu->kvm, NULL, lower_32_bits(val));
0565     if (irq) {
0566         vgic_its_inv_lpi(vcpu->kvm, irq);
0567         vgic_put_irq(vcpu->kvm, irq);
0568     }
0569 
0570     vgic_set_rdist_busy(vcpu, false);
0571 }
0572 
0573 static void vgic_mmio_write_invall(struct kvm_vcpu *vcpu,
0574                    gpa_t addr, unsigned int len,
0575                    unsigned long val)
0576 {
0577     /* See vgic_mmio_write_invlpi() for the early return rationale */
0578     if ((addr & 4) || !vgic_lpis_enabled(vcpu))
0579         return;
0580 
0581     vgic_set_rdist_busy(vcpu, true);
0582     vgic_its_invall(vcpu);
0583     vgic_set_rdist_busy(vcpu, false);
0584 }
0585 
0586 /*
0587  * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the
0588  * redistributors, while SPIs are covered by registers in the distributor
0589  * block. Trying to set private IRQs in this block gets ignored.
0590  * We take some special care here to fix the calculation of the register
0591  * offset.
0592  */
0593 #define REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(off, rd, wr, ur, uw, bpi, acc) \
0594     {                               \
0595         .reg_offset = off,                  \
0596         .bits_per_irq = bpi,                    \
0597         .len = (bpi * VGIC_NR_PRIVATE_IRQS) / 8,        \
0598         .access_flags = acc,                    \
0599         .read = vgic_mmio_read_raz,             \
0600         .write = vgic_mmio_write_wi,                \
0601     }, {                                \
0602         .reg_offset = off + (bpi * VGIC_NR_PRIVATE_IRQS) / 8,   \
0603         .bits_per_irq = bpi,                    \
0604         .len = (bpi * (1024 - VGIC_NR_PRIVATE_IRQS)) / 8,   \
0605         .access_flags = acc,                    \
0606         .read = rd,                     \
0607         .write = wr,                        \
0608         .uaccess_read = ur,                 \
0609         .uaccess_write = uw,                    \
0610     }
0611 
0612 static const struct vgic_register_region vgic_v3_dist_registers[] = {
0613     REGISTER_DESC_WITH_LENGTH_UACCESS(GICD_CTLR,
0614         vgic_mmio_read_v3_misc, vgic_mmio_write_v3_misc,
0615         NULL, vgic_mmio_uaccess_write_v3_misc,
0616         16, VGIC_ACCESS_32bit),
0617     REGISTER_DESC_WITH_LENGTH(GICD_STATUSR,
0618         vgic_mmio_read_rao, vgic_mmio_write_wi, 4,
0619         VGIC_ACCESS_32bit),
0620     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGROUPR,
0621         vgic_mmio_read_group, vgic_mmio_write_group, NULL, NULL, 1,
0622         VGIC_ACCESS_32bit),
0623     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISENABLER,
0624         vgic_mmio_read_enable, vgic_mmio_write_senable,
0625         NULL, vgic_uaccess_write_senable, 1,
0626         VGIC_ACCESS_32bit),
0627     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICENABLER,
0628         vgic_mmio_read_enable, vgic_mmio_write_cenable,
0629            NULL, vgic_uaccess_write_cenable, 1,
0630         VGIC_ACCESS_32bit),
0631     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISPENDR,
0632         vgic_mmio_read_pending, vgic_mmio_write_spending,
0633         vgic_uaccess_read_pending, vgic_v3_uaccess_write_pending, 1,
0634         VGIC_ACCESS_32bit),
0635     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICPENDR,
0636         vgic_mmio_read_pending, vgic_mmio_write_cpending,
0637         vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 1,
0638         VGIC_ACCESS_32bit),
0639     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ISACTIVER,
0640         vgic_mmio_read_active, vgic_mmio_write_sactive,
0641         vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 1,
0642         VGIC_ACCESS_32bit),
0643     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICACTIVER,
0644         vgic_mmio_read_active, vgic_mmio_write_cactive,
0645         vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive,
0646         1, VGIC_ACCESS_32bit),
0647     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IPRIORITYR,
0648         vgic_mmio_read_priority, vgic_mmio_write_priority, NULL, NULL,
0649         8, VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
0650     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ITARGETSR,
0651         vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 8,
0652         VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
0653     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_ICFGR,
0654         vgic_mmio_read_config, vgic_mmio_write_config, NULL, NULL, 2,
0655         VGIC_ACCESS_32bit),
0656     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IGRPMODR,
0657         vgic_mmio_read_raz, vgic_mmio_write_wi, NULL, NULL, 1,
0658         VGIC_ACCESS_32bit),
0659     REGISTER_DESC_WITH_BITS_PER_IRQ_SHARED(GICD_IROUTER,
0660         vgic_mmio_read_irouter, vgic_mmio_write_irouter, NULL, NULL, 64,
0661         VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
0662     REGISTER_DESC_WITH_LENGTH(GICD_IDREGS,
0663         vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
0664         VGIC_ACCESS_32bit),
0665 };
0666 
0667 static const struct vgic_register_region vgic_v3_rd_registers[] = {
0668     /* RD_base registers */
0669     REGISTER_DESC_WITH_LENGTH(GICR_CTLR,
0670         vgic_mmio_read_v3r_ctlr, vgic_mmio_write_v3r_ctlr, 4,
0671         VGIC_ACCESS_32bit),
0672     REGISTER_DESC_WITH_LENGTH(GICR_STATUSR,
0673         vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
0674         VGIC_ACCESS_32bit),
0675     REGISTER_DESC_WITH_LENGTH(GICR_IIDR,
0676         vgic_mmio_read_v3r_iidr, vgic_mmio_write_wi, 4,
0677         VGIC_ACCESS_32bit),
0678     REGISTER_DESC_WITH_LENGTH_UACCESS(GICR_TYPER,
0679         vgic_mmio_read_v3r_typer, vgic_mmio_write_wi,
0680         NULL, vgic_mmio_uaccess_write_wi, 8,
0681         VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
0682     REGISTER_DESC_WITH_LENGTH(GICR_WAKER,
0683         vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
0684         VGIC_ACCESS_32bit),
0685     REGISTER_DESC_WITH_LENGTH(GICR_PROPBASER,
0686         vgic_mmio_read_propbase, vgic_mmio_write_propbase, 8,
0687         VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
0688     REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER,
0689         vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8,
0690         VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
0691     REGISTER_DESC_WITH_LENGTH(GICR_INVLPIR,
0692         vgic_mmio_read_raz, vgic_mmio_write_invlpi, 8,
0693         VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
0694     REGISTER_DESC_WITH_LENGTH(GICR_INVALLR,
0695         vgic_mmio_read_raz, vgic_mmio_write_invall, 8,
0696         VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
0697     REGISTER_DESC_WITH_LENGTH(GICR_SYNCR,
0698         vgic_mmio_read_sync, vgic_mmio_write_wi, 4,
0699         VGIC_ACCESS_32bit),
0700     REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
0701         vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
0702         VGIC_ACCESS_32bit),
0703     /* SGI_base registers */
0704     REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGROUPR0,
0705         vgic_mmio_read_group, vgic_mmio_write_group, 4,
0706         VGIC_ACCESS_32bit),
0707     REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISENABLER0,
0708         vgic_mmio_read_enable, vgic_mmio_write_senable,
0709         NULL, vgic_uaccess_write_senable, 4,
0710         VGIC_ACCESS_32bit),
0711     REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICENABLER0,
0712         vgic_mmio_read_enable, vgic_mmio_write_cenable,
0713         NULL, vgic_uaccess_write_cenable, 4,
0714         VGIC_ACCESS_32bit),
0715     REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISPENDR0,
0716         vgic_mmio_read_pending, vgic_mmio_write_spending,
0717         vgic_uaccess_read_pending, vgic_v3_uaccess_write_pending, 4,
0718         VGIC_ACCESS_32bit),
0719     REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICPENDR0,
0720         vgic_mmio_read_pending, vgic_mmio_write_cpending,
0721         vgic_mmio_read_raz, vgic_mmio_uaccess_write_wi, 4,
0722         VGIC_ACCESS_32bit),
0723     REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ISACTIVER0,
0724         vgic_mmio_read_active, vgic_mmio_write_sactive,
0725         vgic_uaccess_read_active, vgic_mmio_uaccess_write_sactive, 4,
0726         VGIC_ACCESS_32bit),
0727     REGISTER_DESC_WITH_LENGTH_UACCESS(SZ_64K + GICR_ICACTIVER0,
0728         vgic_mmio_read_active, vgic_mmio_write_cactive,
0729         vgic_uaccess_read_active, vgic_mmio_uaccess_write_cactive, 4,
0730         VGIC_ACCESS_32bit),
0731     REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IPRIORITYR0,
0732         vgic_mmio_read_priority, vgic_mmio_write_priority, 32,
0733         VGIC_ACCESS_32bit | VGIC_ACCESS_8bit),
0734     REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_ICFGR0,
0735         vgic_mmio_read_config, vgic_mmio_write_config, 8,
0736         VGIC_ACCESS_32bit),
0737     REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_IGRPMODR0,
0738         vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
0739         VGIC_ACCESS_32bit),
0740     REGISTER_DESC_WITH_LENGTH(SZ_64K + GICR_NSACR,
0741         vgic_mmio_read_raz, vgic_mmio_write_wi, 4,
0742         VGIC_ACCESS_32bit),
0743 };
0744 
0745 unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev)
0746 {
0747     dev->regions = vgic_v3_dist_registers;
0748     dev->nr_regions = ARRAY_SIZE(vgic_v3_dist_registers);
0749 
0750     kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops);
0751 
0752     return SZ_64K;
0753 }
0754 
0755 /**
0756  * vgic_register_redist_iodev - register a single redist iodev
0757  * @vcpu:    The VCPU to which the redistributor belongs
0758  *
0759  * Register a KVM iodev for this VCPU's redistributor using the address
0760  * provided.
0761  *
0762  * Return 0 on success, -ERRNO otherwise.
0763  */
0764 int vgic_register_redist_iodev(struct kvm_vcpu *vcpu)
0765 {
0766     struct kvm *kvm = vcpu->kvm;
0767     struct vgic_dist *vgic = &kvm->arch.vgic;
0768     struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
0769     struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
0770     struct vgic_redist_region *rdreg;
0771     gpa_t rd_base;
0772     int ret;
0773 
0774     if (!IS_VGIC_ADDR_UNDEF(vgic_cpu->rd_iodev.base_addr))
0775         return 0;
0776 
0777     /*
0778      * We may be creating VCPUs before having set the base address for the
0779      * redistributor region, in which case we will come back to this
0780      * function for all VCPUs when the base address is set.  Just return
0781      * without doing any work for now.
0782      */
0783     rdreg = vgic_v3_rdist_free_slot(&vgic->rd_regions);
0784     if (!rdreg)
0785         return 0;
0786 
0787     if (!vgic_v3_check_base(kvm))
0788         return -EINVAL;
0789 
0790     vgic_cpu->rdreg = rdreg;
0791     vgic_cpu->rdreg_index = rdreg->free_index;
0792 
0793     rd_base = rdreg->base + rdreg->free_index * KVM_VGIC_V3_REDIST_SIZE;
0794 
0795     kvm_iodevice_init(&rd_dev->dev, &kvm_io_gic_ops);
0796     rd_dev->base_addr = rd_base;
0797     rd_dev->iodev_type = IODEV_REDIST;
0798     rd_dev->regions = vgic_v3_rd_registers;
0799     rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
0800     rd_dev->redist_vcpu = vcpu;
0801 
0802     mutex_lock(&kvm->slots_lock);
0803     ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,
0804                       2 * SZ_64K, &rd_dev->dev);
0805     mutex_unlock(&kvm->slots_lock);
0806 
0807     if (ret)
0808         return ret;
0809 
0810     rdreg->free_index++;
0811     return 0;
0812 }
0813 
0814 static void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu)
0815 {
0816     struct vgic_io_device *rd_dev = &vcpu->arch.vgic_cpu.rd_iodev;
0817 
0818     kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);
0819 }
0820 
0821 static int vgic_register_all_redist_iodevs(struct kvm *kvm)
0822 {
0823     struct kvm_vcpu *vcpu;
0824     unsigned long c;
0825     int ret = 0;
0826 
0827     kvm_for_each_vcpu(c, vcpu, kvm) {
0828         ret = vgic_register_redist_iodev(vcpu);
0829         if (ret)
0830             break;
0831     }
0832 
0833     if (ret) {
0834         /* The current c failed, so iterate over the previous ones. */
0835         int i;
0836 
0837         mutex_lock(&kvm->slots_lock);
0838         for (i = 0; i < c; i++) {
0839             vcpu = kvm_get_vcpu(kvm, i);
0840             vgic_unregister_redist_iodev(vcpu);
0841         }
0842         mutex_unlock(&kvm->slots_lock);
0843     }
0844 
0845     return ret;
0846 }
0847 
0848 /**
0849  * vgic_v3_alloc_redist_region - Allocate a new redistributor region
0850  *
0851  * Performs various checks before inserting the rdist region in the list.
0852  * Those tests depend on whether the size of the rdist region is known
0853  * (ie. count != 0). The list is sorted by rdist region index.
0854  *
0855  * @kvm: kvm handle
0856  * @index: redist region index
0857  * @base: base of the new rdist region
0858  * @count: number of redistributors the region is made of (0 in the old style
0859  * single region, whose size is induced from the number of vcpus)
0860  *
0861  * Return 0 on success, < 0 otherwise
0862  */
0863 static int vgic_v3_alloc_redist_region(struct kvm *kvm, uint32_t index,
0864                        gpa_t base, uint32_t count)
0865 {
0866     struct vgic_dist *d = &kvm->arch.vgic;
0867     struct vgic_redist_region *rdreg;
0868     struct list_head *rd_regions = &d->rd_regions;
0869     int nr_vcpus = atomic_read(&kvm->online_vcpus);
0870     size_t size = count ? count * KVM_VGIC_V3_REDIST_SIZE
0871                 : nr_vcpus * KVM_VGIC_V3_REDIST_SIZE;
0872     int ret;
0873 
0874     /* cross the end of memory ? */
0875     if (base + size < base)
0876         return -EINVAL;
0877 
0878     if (list_empty(rd_regions)) {
0879         if (index != 0)
0880             return -EINVAL;
0881     } else {
0882         rdreg = list_last_entry(rd_regions,
0883                     struct vgic_redist_region, list);
0884 
0885         /* Don't mix single region and discrete redist regions */
0886         if (!count && rdreg->count)
0887             return -EINVAL;
0888 
0889         if (!count)
0890             return -EEXIST;
0891 
0892         if (index != rdreg->index + 1)
0893             return -EINVAL;
0894     }
0895 
0896     /*
0897      * For legacy single-region redistributor regions (!count),
0898      * check that the redistributor region does not overlap with the
0899      * distributor's address space.
0900      */
0901     if (!count && !IS_VGIC_ADDR_UNDEF(d->vgic_dist_base) &&
0902         vgic_dist_overlap(kvm, base, size))
0903         return -EINVAL;
0904 
0905     /* collision with any other rdist region? */
0906     if (vgic_v3_rdist_overlap(kvm, base, size))
0907         return -EINVAL;
0908 
0909     rdreg = kzalloc(sizeof(*rdreg), GFP_KERNEL_ACCOUNT);
0910     if (!rdreg)
0911         return -ENOMEM;
0912 
0913     rdreg->base = VGIC_ADDR_UNDEF;
0914 
0915     ret = vgic_check_iorange(kvm, rdreg->base, base, SZ_64K, size);
0916     if (ret)
0917         goto free;
0918 
0919     rdreg->base = base;
0920     rdreg->count = count;
0921     rdreg->free_index = 0;
0922     rdreg->index = index;
0923 
0924     list_add_tail(&rdreg->list, rd_regions);
0925     return 0;
0926 free:
0927     kfree(rdreg);
0928     return ret;
0929 }
0930 
0931 void vgic_v3_free_redist_region(struct vgic_redist_region *rdreg)
0932 {
0933     list_del(&rdreg->list);
0934     kfree(rdreg);
0935 }
0936 
0937 int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count)
0938 {
0939     int ret;
0940 
0941     ret = vgic_v3_alloc_redist_region(kvm, index, addr, count);
0942     if (ret)
0943         return ret;
0944 
0945     /*
0946      * Register iodevs for each existing VCPU.  Adding more VCPUs
0947      * afterwards will register the iodevs when needed.
0948      */
0949     ret = vgic_register_all_redist_iodevs(kvm);
0950     if (ret) {
0951         struct vgic_redist_region *rdreg;
0952 
0953         rdreg = vgic_v3_rdist_region_from_index(kvm, index);
0954         vgic_v3_free_redist_region(rdreg);
0955         return ret;
0956     }
0957 
0958     return 0;
0959 }
0960 
0961 int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr)
0962 {
0963     const struct vgic_register_region *region;
0964     struct vgic_io_device iodev;
0965     struct vgic_reg_attr reg_attr;
0966     struct kvm_vcpu *vcpu;
0967     gpa_t addr;
0968     int ret;
0969 
0970     ret = vgic_v3_parse_attr(dev, attr, &reg_attr);
0971     if (ret)
0972         return ret;
0973 
0974     vcpu = reg_attr.vcpu;
0975     addr = reg_attr.addr;
0976 
0977     switch (attr->group) {
0978     case KVM_DEV_ARM_VGIC_GRP_DIST_REGS:
0979         iodev.regions = vgic_v3_dist_registers;
0980         iodev.nr_regions = ARRAY_SIZE(vgic_v3_dist_registers);
0981         iodev.base_addr = 0;
0982         break;
0983     case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:{
0984         iodev.regions = vgic_v3_rd_registers;
0985         iodev.nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
0986         iodev.base_addr = 0;
0987         break;
0988     }
0989     case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
0990         return vgic_v3_has_cpu_sysregs_attr(vcpu, attr);
0991     default:
0992         return -ENXIO;
0993     }
0994 
0995     /* We only support aligned 32-bit accesses. */
0996     if (addr & 3)
0997         return -ENXIO;
0998 
0999     region = vgic_get_mmio_region(vcpu, &iodev, addr, sizeof(u32));
1000     if (!region)
1001         return -ENXIO;
1002 
1003     return 0;
1004 }
1005 /*
1006  * Compare a given affinity (level 1-3 and a level 0 mask, from the SGI
1007  * generation register ICC_SGI1R_EL1) with a given VCPU.
1008  * If the VCPU's MPIDR matches, return the level0 affinity, otherwise
1009  * return -1.
1010  */
1011 static int match_mpidr(u64 sgi_aff, u16 sgi_cpu_mask, struct kvm_vcpu *vcpu)
1012 {
1013     unsigned long affinity;
1014     int level0;
1015 
1016     /*
1017      * Split the current VCPU's MPIDR into affinity level 0 and the
1018      * rest as this is what we have to compare against.
1019      */
1020     affinity = kvm_vcpu_get_mpidr_aff(vcpu);
1021     level0 = MPIDR_AFFINITY_LEVEL(affinity, 0);
1022     affinity &= ~MPIDR_LEVEL_MASK;
1023 
1024     /* bail out if the upper three levels don't match */
1025     if (sgi_aff != affinity)
1026         return -1;
1027 
1028     /* Is this VCPU's bit set in the mask ? */
1029     if (!(sgi_cpu_mask & BIT(level0)))
1030         return -1;
1031 
1032     return level0;
1033 }
1034 
1035 /*
1036  * The ICC_SGI* registers encode the affinity differently from the MPIDR,
1037  * so provide a wrapper to use the existing defines to isolate a certain
1038  * affinity level.
1039  */
1040 #define SGI_AFFINITY_LEVEL(reg, level) \
1041     ((((reg) & ICC_SGI1R_AFFINITY_## level ##_MASK) \
1042     >> ICC_SGI1R_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level))
1043 
1044 /**
1045  * vgic_v3_dispatch_sgi - handle SGI requests from VCPUs
1046  * @vcpu: The VCPU requesting a SGI
1047  * @reg: The value written into ICC_{ASGI1,SGI0,SGI1}R by that VCPU
1048  * @allow_group1: Does the sysreg access allow generation of G1 SGIs
1049  *
1050  * With GICv3 (and ARE=1) CPUs trigger SGIs by writing to a system register.
1051  * This will trap in sys_regs.c and call this function.
1052  * This ICC_SGI1R_EL1 register contains the upper three affinity levels of the
1053  * target processors as well as a bitmask of 16 Aff0 CPUs.
1054  * If the interrupt routing mode bit is not set, we iterate over all VCPUs to
1055  * check for matching ones. If this bit is set, we signal all, but not the
1056  * calling VCPU.
1057  */
1058 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1)
1059 {
1060     struct kvm *kvm = vcpu->kvm;
1061     struct kvm_vcpu *c_vcpu;
1062     u16 target_cpus;
1063     u64 mpidr;
1064     int sgi;
1065     int vcpu_id = vcpu->vcpu_id;
1066     bool broadcast;
1067     unsigned long c, flags;
1068 
1069     sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT;
1070     broadcast = reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT);
1071     target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT;
1072     mpidr = SGI_AFFINITY_LEVEL(reg, 3);
1073     mpidr |= SGI_AFFINITY_LEVEL(reg, 2);
1074     mpidr |= SGI_AFFINITY_LEVEL(reg, 1);
1075 
1076     /*
1077      * We iterate over all VCPUs to find the MPIDRs matching the request.
1078      * If we have handled one CPU, we clear its bit to detect early
1079      * if we are already finished. This avoids iterating through all
1080      * VCPUs when most of the times we just signal a single VCPU.
1081      */
1082     kvm_for_each_vcpu(c, c_vcpu, kvm) {
1083         struct vgic_irq *irq;
1084 
1085         /* Exit early if we have dealt with all requested CPUs */
1086         if (!broadcast && target_cpus == 0)
1087             break;
1088 
1089         /* Don't signal the calling VCPU */
1090         if (broadcast && c == vcpu_id)
1091             continue;
1092 
1093         if (!broadcast) {
1094             int level0;
1095 
1096             level0 = match_mpidr(mpidr, target_cpus, c_vcpu);
1097             if (level0 == -1)
1098                 continue;
1099 
1100             /* remove this matching VCPU from the mask */
1101             target_cpus &= ~BIT(level0);
1102         }
1103 
1104         irq = vgic_get_irq(vcpu->kvm, c_vcpu, sgi);
1105 
1106         raw_spin_lock_irqsave(&irq->irq_lock, flags);
1107 
1108         /*
1109          * An access targeting Group0 SGIs can only generate
1110          * those, while an access targeting Group1 SGIs can
1111          * generate interrupts of either group.
1112          */
1113         if (!irq->group || allow_group1) {
1114             if (!irq->hw) {
1115                 irq->pending_latch = true;
1116                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
1117             } else {
1118                 /* HW SGI? Ask the GIC to inject it */
1119                 int err;
1120                 err = irq_set_irqchip_state(irq->host_irq,
1121                                 IRQCHIP_STATE_PENDING,
1122                                 true);
1123                 WARN_RATELIMIT(err, "IRQ %d", irq->host_irq);
1124                 raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1125             }
1126         } else {
1127             raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
1128         }
1129 
1130         vgic_put_irq(vcpu->kvm, irq);
1131     }
1132 }
1133 
1134 int vgic_v3_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1135              int offset, u32 *val)
1136 {
1137     struct vgic_io_device dev = {
1138         .regions = vgic_v3_dist_registers,
1139         .nr_regions = ARRAY_SIZE(vgic_v3_dist_registers),
1140     };
1141 
1142     return vgic_uaccess(vcpu, &dev, is_write, offset, val);
1143 }
1144 
1145 int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1146                int offset, u32 *val)
1147 {
1148     struct vgic_io_device rd_dev = {
1149         .regions = vgic_v3_rd_registers,
1150         .nr_regions = ARRAY_SIZE(vgic_v3_rd_registers),
1151     };
1152 
1153     return vgic_uaccess(vcpu, &rd_dev, is_write, offset, val);
1154 }
1155 
1156 int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,
1157                     u32 intid, u32 *val)
1158 {
1159     if (intid % 32)
1160         return -EINVAL;
1161 
1162     if (is_write)
1163         vgic_write_irq_line_level_info(vcpu, intid, *val);
1164     else
1165         *val = vgic_read_irq_line_level_info(vcpu, intid);
1166 
1167     return 0;
1168 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!