Back to home page

OSCL-LXR
 
 

     

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2012-2015 - ARM Ltd
0004  * Author: Marc Zyngier <marc.zyngier@arm.com>
0005  */
0006 
0007 #include <hyp/adjust_pc.h>
0008 
0009 #include <linux/compiler.h>
0010 #include <linux/irqchip/arm-gic-v3.h>
0011 #include <linux/kvm_host.h>
0012 
0013 #include <asm/kvm_emulate.h>
0014 #include <asm/kvm_hyp.h>
0015 #include <asm/kvm_mmu.h>
0016 
0017 #define vtr_to_max_lr_idx(v)        ((v) & 0xf)
0018 #define vtr_to_nr_pre_bits(v)       ((((u32)(v) >> 26) & 7) + 1)
0019 #define vtr_to_nr_apr_regs(v)       (1 << (vtr_to_nr_pre_bits(v) - 5))
0020 
0021 static u64 __gic_v3_get_lr(unsigned int lr)
0022 {
0023     switch (lr & 0xf) {
0024     case 0:
0025         return read_gicreg(ICH_LR0_EL2);
0026     case 1:
0027         return read_gicreg(ICH_LR1_EL2);
0028     case 2:
0029         return read_gicreg(ICH_LR2_EL2);
0030     case 3:
0031         return read_gicreg(ICH_LR3_EL2);
0032     case 4:
0033         return read_gicreg(ICH_LR4_EL2);
0034     case 5:
0035         return read_gicreg(ICH_LR5_EL2);
0036     case 6:
0037         return read_gicreg(ICH_LR6_EL2);
0038     case 7:
0039         return read_gicreg(ICH_LR7_EL2);
0040     case 8:
0041         return read_gicreg(ICH_LR8_EL2);
0042     case 9:
0043         return read_gicreg(ICH_LR9_EL2);
0044     case 10:
0045         return read_gicreg(ICH_LR10_EL2);
0046     case 11:
0047         return read_gicreg(ICH_LR11_EL2);
0048     case 12:
0049         return read_gicreg(ICH_LR12_EL2);
0050     case 13:
0051         return read_gicreg(ICH_LR13_EL2);
0052     case 14:
0053         return read_gicreg(ICH_LR14_EL2);
0054     case 15:
0055         return read_gicreg(ICH_LR15_EL2);
0056     }
0057 
0058     unreachable();
0059 }
0060 
0061 static void __gic_v3_set_lr(u64 val, int lr)
0062 {
0063     switch (lr & 0xf) {
0064     case 0:
0065         write_gicreg(val, ICH_LR0_EL2);
0066         break;
0067     case 1:
0068         write_gicreg(val, ICH_LR1_EL2);
0069         break;
0070     case 2:
0071         write_gicreg(val, ICH_LR2_EL2);
0072         break;
0073     case 3:
0074         write_gicreg(val, ICH_LR3_EL2);
0075         break;
0076     case 4:
0077         write_gicreg(val, ICH_LR4_EL2);
0078         break;
0079     case 5:
0080         write_gicreg(val, ICH_LR5_EL2);
0081         break;
0082     case 6:
0083         write_gicreg(val, ICH_LR6_EL2);
0084         break;
0085     case 7:
0086         write_gicreg(val, ICH_LR7_EL2);
0087         break;
0088     case 8:
0089         write_gicreg(val, ICH_LR8_EL2);
0090         break;
0091     case 9:
0092         write_gicreg(val, ICH_LR9_EL2);
0093         break;
0094     case 10:
0095         write_gicreg(val, ICH_LR10_EL2);
0096         break;
0097     case 11:
0098         write_gicreg(val, ICH_LR11_EL2);
0099         break;
0100     case 12:
0101         write_gicreg(val, ICH_LR12_EL2);
0102         break;
0103     case 13:
0104         write_gicreg(val, ICH_LR13_EL2);
0105         break;
0106     case 14:
0107         write_gicreg(val, ICH_LR14_EL2);
0108         break;
0109     case 15:
0110         write_gicreg(val, ICH_LR15_EL2);
0111         break;
0112     }
0113 }
0114 
0115 static void __vgic_v3_write_ap0rn(u32 val, int n)
0116 {
0117     switch (n) {
0118     case 0:
0119         write_gicreg(val, ICH_AP0R0_EL2);
0120         break;
0121     case 1:
0122         write_gicreg(val, ICH_AP0R1_EL2);
0123         break;
0124     case 2:
0125         write_gicreg(val, ICH_AP0R2_EL2);
0126         break;
0127     case 3:
0128         write_gicreg(val, ICH_AP0R3_EL2);
0129         break;
0130     }
0131 }
0132 
0133 static void __vgic_v3_write_ap1rn(u32 val, int n)
0134 {
0135     switch (n) {
0136     case 0:
0137         write_gicreg(val, ICH_AP1R0_EL2);
0138         break;
0139     case 1:
0140         write_gicreg(val, ICH_AP1R1_EL2);
0141         break;
0142     case 2:
0143         write_gicreg(val, ICH_AP1R2_EL2);
0144         break;
0145     case 3:
0146         write_gicreg(val, ICH_AP1R3_EL2);
0147         break;
0148     }
0149 }
0150 
0151 static u32 __vgic_v3_read_ap0rn(int n)
0152 {
0153     u32 val;
0154 
0155     switch (n) {
0156     case 0:
0157         val = read_gicreg(ICH_AP0R0_EL2);
0158         break;
0159     case 1:
0160         val = read_gicreg(ICH_AP0R1_EL2);
0161         break;
0162     case 2:
0163         val = read_gicreg(ICH_AP0R2_EL2);
0164         break;
0165     case 3:
0166         val = read_gicreg(ICH_AP0R3_EL2);
0167         break;
0168     default:
0169         unreachable();
0170     }
0171 
0172     return val;
0173 }
0174 
0175 static u32 __vgic_v3_read_ap1rn(int n)
0176 {
0177     u32 val;
0178 
0179     switch (n) {
0180     case 0:
0181         val = read_gicreg(ICH_AP1R0_EL2);
0182         break;
0183     case 1:
0184         val = read_gicreg(ICH_AP1R1_EL2);
0185         break;
0186     case 2:
0187         val = read_gicreg(ICH_AP1R2_EL2);
0188         break;
0189     case 3:
0190         val = read_gicreg(ICH_AP1R3_EL2);
0191         break;
0192     default:
0193         unreachable();
0194     }
0195 
0196     return val;
0197 }
0198 
0199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
0200 {
0201     u64 used_lrs = cpu_if->used_lrs;
0202 
0203     /*
0204      * Make sure stores to the GIC via the memory mapped interface
0205      * are now visible to the system register interface when reading the
0206      * LRs, and when reading back the VMCR on non-VHE systems.
0207      */
0208     if (used_lrs || !has_vhe()) {
0209         if (!cpu_if->vgic_sre) {
0210             dsb(sy);
0211             isb();
0212         }
0213     }
0214 
0215     if (used_lrs || cpu_if->its_vpe.its_vm) {
0216         int i;
0217         u32 elrsr;
0218 
0219         elrsr = read_gicreg(ICH_ELRSR_EL2);
0220 
0221         write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2);
0222 
0223         for (i = 0; i < used_lrs; i++) {
0224             if (elrsr & (1 << i))
0225                 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
0226             else
0227                 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
0228 
0229             __gic_v3_set_lr(0, i);
0230         }
0231     }
0232 }
0233 
0234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
0235 {
0236     u64 used_lrs = cpu_if->used_lrs;
0237     int i;
0238 
0239     if (used_lrs || cpu_if->its_vpe.its_vm) {
0240         write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
0241 
0242         for (i = 0; i < used_lrs; i++)
0243             __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
0244     }
0245 
0246     /*
0247      * Ensure that writes to the LRs, and on non-VHE systems ensure that
0248      * the write to the VMCR in __vgic_v3_activate_traps(), will have
0249      * reached the (re)distributors. This ensure the guest will read the
0250      * correct values from the memory-mapped interface.
0251      */
0252     if (used_lrs || !has_vhe()) {
0253         if (!cpu_if->vgic_sre) {
0254             isb();
0255             dsb(sy);
0256         }
0257     }
0258 }
0259 
0260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
0261 {
0262     /*
0263      * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
0264      * Group0 interrupt (as generated in GICv2 mode) to be
0265      * delivered as a FIQ to the guest, with potentially fatal
0266      * consequences. So we must make sure that ICC_SRE_EL1 has
0267      * been actually programmed with the value we want before
0268      * starting to mess with the rest of the GIC, and VMCR_EL2 in
0269      * particular.  This logic must be called before
0270      * __vgic_v3_restore_state().
0271      */
0272     if (!cpu_if->vgic_sre) {
0273         write_gicreg(0, ICC_SRE_EL1);
0274         isb();
0275         write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
0276 
0277 
0278         if (has_vhe()) {
0279             /*
0280              * Ensure that the write to the VMCR will have reached
0281              * the (re)distributors. This ensure the guest will
0282              * read the correct values from the memory-mapped
0283              * interface.
0284              */
0285             isb();
0286             dsb(sy);
0287         }
0288     }
0289 
0290     /*
0291      * Prevent the guest from touching the GIC system registers if
0292      * SRE isn't enabled for GICv3 emulation.
0293      */
0294     write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
0295              ICC_SRE_EL2);
0296 
0297     /*
0298      * If we need to trap system registers, we must write
0299      * ICH_HCR_EL2 anyway, even if no interrupts are being
0300      * injected,
0301      */
0302     if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
0303         cpu_if->its_vpe.its_vm)
0304         write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
0305 }
0306 
0307 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
0308 {
0309     u64 val;
0310 
0311     if (!cpu_if->vgic_sre) {
0312         cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
0313     }
0314 
0315     val = read_gicreg(ICC_SRE_EL2);
0316     write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
0317 
0318     if (!cpu_if->vgic_sre) {
0319         /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
0320         isb();
0321         write_gicreg(1, ICC_SRE_EL1);
0322     }
0323 
0324     /*
0325      * If we were trapping system registers, we enabled the VGIC even if
0326      * no interrupts were being injected, and we disable it again here.
0327      */
0328     if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
0329         cpu_if->its_vpe.its_vm)
0330         write_gicreg(0, ICH_HCR_EL2);
0331 }
0332 
0333 void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
0334 {
0335     u64 val;
0336     u32 nr_pre_bits;
0337 
0338     val = read_gicreg(ICH_VTR_EL2);
0339     nr_pre_bits = vtr_to_nr_pre_bits(val);
0340 
0341     switch (nr_pre_bits) {
0342     case 7:
0343         cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
0344         cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
0345         fallthrough;
0346     case 6:
0347         cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
0348         fallthrough;
0349     default:
0350         cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
0351     }
0352 
0353     switch (nr_pre_bits) {
0354     case 7:
0355         cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
0356         cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
0357         fallthrough;
0358     case 6:
0359         cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
0360         fallthrough;
0361     default:
0362         cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
0363     }
0364 }
0365 
0366 void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
0367 {
0368     u64 val;
0369     u32 nr_pre_bits;
0370 
0371     val = read_gicreg(ICH_VTR_EL2);
0372     nr_pre_bits = vtr_to_nr_pre_bits(val);
0373 
0374     switch (nr_pre_bits) {
0375     case 7:
0376         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
0377         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
0378         fallthrough;
0379     case 6:
0380         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
0381         fallthrough;
0382     default:
0383         __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
0384     }
0385 
0386     switch (nr_pre_bits) {
0387     case 7:
0388         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
0389         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
0390         fallthrough;
0391     case 6:
0392         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
0393         fallthrough;
0394     default:
0395         __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
0396     }
0397 }
0398 
0399 void __vgic_v3_init_lrs(void)
0400 {
0401     int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
0402     int i;
0403 
0404     for (i = 0; i <= max_lr_idx; i++)
0405         __gic_v3_set_lr(0, i);
0406 }
0407 
0408 /*
0409  * Return the GIC CPU configuration:
0410  * - [31:0]  ICH_VTR_EL2
0411  * - [62:32] RES0
0412  * - [63]    MMIO (GICv2) capable
0413  */
0414 u64 __vgic_v3_get_gic_config(void)
0415 {
0416     u64 val, sre = read_gicreg(ICC_SRE_EL1);
0417     unsigned long flags = 0;
0418 
0419     /*
0420      * To check whether we have a MMIO-based (GICv2 compatible)
0421      * CPU interface, we need to disable the system register
0422      * view. To do that safely, we have to prevent any interrupt
0423      * from firing (which would be deadly).
0424      *
0425      * Note that this only makes sense on VHE, as interrupts are
0426      * already masked for nVHE as part of the exception entry to
0427      * EL2.
0428      */
0429     if (has_vhe())
0430         flags = local_daif_save();
0431 
0432     /*
0433      * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
0434      * that to be able to set ICC_SRE_EL1.SRE to 0, all the
0435      * interrupt overrides must be set. You've got to love this.
0436      */
0437     sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO);
0438     isb();
0439     write_gicreg(0, ICC_SRE_EL1);
0440     isb();
0441 
0442     val = read_gicreg(ICC_SRE_EL1);
0443 
0444     write_gicreg(sre, ICC_SRE_EL1);
0445     isb();
0446     sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0);
0447     isb();
0448 
0449     if (has_vhe())
0450         local_daif_restore(flags);
0451 
0452     val  = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
0453     val |= read_gicreg(ICH_VTR_EL2);
0454 
0455     return val;
0456 }
0457 
0458 u64 __vgic_v3_read_vmcr(void)
0459 {
0460     return read_gicreg(ICH_VMCR_EL2);
0461 }
0462 
0463 void __vgic_v3_write_vmcr(u32 vmcr)
0464 {
0465     write_gicreg(vmcr, ICH_VMCR_EL2);
0466 }
0467 
0468 static int __vgic_v3_bpr_min(void)
0469 {
0470     /* See Pseudocode for VPriorityGroup */
0471     return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
0472 }
0473 
0474 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
0475 {
0476     u64 esr = kvm_vcpu_get_esr(vcpu);
0477     u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
0478 
0479     return crm != 8;
0480 }
0481 
0482 #define GICv3_IDLE_PRIORITY 0xff
0483 
0484 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
0485                      u64 *lr_val)
0486 {
0487     unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
0488     u8 priority = GICv3_IDLE_PRIORITY;
0489     int i, lr = -1;
0490 
0491     for (i = 0; i < used_lrs; i++) {
0492         u64 val = __gic_v3_get_lr(i);
0493         u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
0494 
0495         /* Not pending in the state? */
0496         if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
0497             continue;
0498 
0499         /* Group-0 interrupt, but Group-0 disabled? */
0500         if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
0501             continue;
0502 
0503         /* Group-1 interrupt, but Group-1 disabled? */
0504         if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
0505             continue;
0506 
0507         /* Not the highest priority? */
0508         if (lr_prio >= priority)
0509             continue;
0510 
0511         /* This is a candidate */
0512         priority = lr_prio;
0513         *lr_val = val;
0514         lr = i;
0515     }
0516 
0517     if (lr == -1)
0518         *lr_val = ICC_IAR1_EL1_SPURIOUS;
0519 
0520     return lr;
0521 }
0522 
0523 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
0524                     u64 *lr_val)
0525 {
0526     unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
0527     int i;
0528 
0529     for (i = 0; i < used_lrs; i++) {
0530         u64 val = __gic_v3_get_lr(i);
0531 
0532         if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
0533             (val & ICH_LR_ACTIVE_BIT)) {
0534             *lr_val = val;
0535             return i;
0536         }
0537     }
0538 
0539     *lr_val = ICC_IAR1_EL1_SPURIOUS;
0540     return -1;
0541 }
0542 
0543 static int __vgic_v3_get_highest_active_priority(void)
0544 {
0545     u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
0546     u32 hap = 0;
0547     int i;
0548 
0549     for (i = 0; i < nr_apr_regs; i++) {
0550         u32 val;
0551 
0552         /*
0553          * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
0554          * contain the active priority levels for this VCPU
0555          * for the maximum number of supported priority
0556          * levels, and we return the full priority level only
0557          * if the BPR is programmed to its minimum, otherwise
0558          * we return a combination of the priority level and
0559          * subpriority, as determined by the setting of the
0560          * BPR, but without the full subpriority.
0561          */
0562         val  = __vgic_v3_read_ap0rn(i);
0563         val |= __vgic_v3_read_ap1rn(i);
0564         if (!val) {
0565             hap += 32;
0566             continue;
0567         }
0568 
0569         return (hap + __ffs(val)) << __vgic_v3_bpr_min();
0570     }
0571 
0572     return GICv3_IDLE_PRIORITY;
0573 }
0574 
0575 static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
0576 {
0577     return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
0578 }
0579 
0580 static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
0581 {
0582     unsigned int bpr;
0583 
0584     if (vmcr & ICH_VMCR_CBPR_MASK) {
0585         bpr = __vgic_v3_get_bpr0(vmcr);
0586         if (bpr < 7)
0587             bpr++;
0588     } else {
0589         bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
0590     }
0591 
0592     return bpr;
0593 }
0594 
0595 /*
0596  * Convert a priority to a preemption level, taking the relevant BPR
0597  * into account by zeroing the sub-priority bits.
0598  */
0599 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
0600 {
0601     unsigned int bpr;
0602 
0603     if (!grp)
0604         bpr = __vgic_v3_get_bpr0(vmcr) + 1;
0605     else
0606         bpr = __vgic_v3_get_bpr1(vmcr);
0607 
0608     return pri & (GENMASK(7, 0) << bpr);
0609 }
0610 
0611 /*
0612  * The priority value is independent of any of the BPR values, so we
0613  * normalize it using the minimal BPR value. This guarantees that no
0614  * matter what the guest does with its BPR, we can always set/get the
0615  * same value of a priority.
0616  */
0617 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
0618 {
0619     u8 pre, ap;
0620     u32 val;
0621     int apr;
0622 
0623     pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
0624     ap = pre >> __vgic_v3_bpr_min();
0625     apr = ap / 32;
0626 
0627     if (!grp) {
0628         val = __vgic_v3_read_ap0rn(apr);
0629         __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
0630     } else {
0631         val = __vgic_v3_read_ap1rn(apr);
0632         __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
0633     }
0634 }
0635 
0636 static int __vgic_v3_clear_highest_active_priority(void)
0637 {
0638     u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
0639     u32 hap = 0;
0640     int i;
0641 
0642     for (i = 0; i < nr_apr_regs; i++) {
0643         u32 ap0, ap1;
0644         int c0, c1;
0645 
0646         ap0 = __vgic_v3_read_ap0rn(i);
0647         ap1 = __vgic_v3_read_ap1rn(i);
0648         if (!ap0 && !ap1) {
0649             hap += 32;
0650             continue;
0651         }
0652 
0653         c0 = ap0 ? __ffs(ap0) : 32;
0654         c1 = ap1 ? __ffs(ap1) : 32;
0655 
0656         /* Always clear the LSB, which is the highest priority */
0657         if (c0 < c1) {
0658             ap0 &= ~BIT(c0);
0659             __vgic_v3_write_ap0rn(ap0, i);
0660             hap += c0;
0661         } else {
0662             ap1 &= ~BIT(c1);
0663             __vgic_v3_write_ap1rn(ap1, i);
0664             hap += c1;
0665         }
0666 
0667         /* Rescale to 8 bits of priority */
0668         return hap << __vgic_v3_bpr_min();
0669     }
0670 
0671     return GICv3_IDLE_PRIORITY;
0672 }
0673 
0674 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0675 {
0676     u64 lr_val;
0677     u8 lr_prio, pmr;
0678     int lr, grp;
0679 
0680     grp = __vgic_v3_get_group(vcpu);
0681 
0682     lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
0683     if (lr < 0)
0684         goto spurious;
0685 
0686     if (grp != !!(lr_val & ICH_LR_GROUP))
0687         goto spurious;
0688 
0689     pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
0690     lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
0691     if (pmr <= lr_prio)
0692         goto spurious;
0693 
0694     if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
0695         goto spurious;
0696 
0697     lr_val &= ~ICH_LR_STATE;
0698     lr_val |= ICH_LR_ACTIVE_BIT;
0699     __gic_v3_set_lr(lr_val, lr);
0700     __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
0701     vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
0702     return;
0703 
0704 spurious:
0705     vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
0706 }
0707 
0708 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
0709 {
0710     lr_val &= ~ICH_LR_ACTIVE_BIT;
0711     if (lr_val & ICH_LR_HW) {
0712         u32 pid;
0713 
0714         pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
0715         gic_write_dir(pid);
0716     }
0717 
0718     __gic_v3_set_lr(lr_val, lr);
0719 }
0720 
0721 static void __vgic_v3_bump_eoicount(void)
0722 {
0723     u32 hcr;
0724 
0725     hcr = read_gicreg(ICH_HCR_EL2);
0726     hcr += 1 << ICH_HCR_EOIcount_SHIFT;
0727     write_gicreg(hcr, ICH_HCR_EL2);
0728 }
0729 
0730 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0731 {
0732     u32 vid = vcpu_get_reg(vcpu, rt);
0733     u64 lr_val;
0734     int lr;
0735 
0736     /* EOImode == 0, nothing to be done here */
0737     if (!(vmcr & ICH_VMCR_EOIM_MASK))
0738         return;
0739 
0740     /* No deactivate to be performed on an LPI */
0741     if (vid >= VGIC_MIN_LPI)
0742         return;
0743 
0744     lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
0745     if (lr == -1) {
0746         __vgic_v3_bump_eoicount();
0747         return;
0748     }
0749 
0750     __vgic_v3_clear_active_lr(lr, lr_val);
0751 }
0752 
0753 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0754 {
0755     u32 vid = vcpu_get_reg(vcpu, rt);
0756     u64 lr_val;
0757     u8 lr_prio, act_prio;
0758     int lr, grp;
0759 
0760     grp = __vgic_v3_get_group(vcpu);
0761 
0762     /* Drop priority in any case */
0763     act_prio = __vgic_v3_clear_highest_active_priority();
0764 
0765     lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
0766     if (lr == -1) {
0767         /* Do not bump EOIcount for LPIs that aren't in the LRs */
0768         if (!(vid >= VGIC_MIN_LPI))
0769             __vgic_v3_bump_eoicount();
0770         return;
0771     }
0772 
0773     /* EOImode == 1 and not an LPI, nothing to be done here */
0774     if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
0775         return;
0776 
0777     lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
0778 
0779     /* If priorities or group do not match, the guest has fscked-up. */
0780     if (grp != !!(lr_val & ICH_LR_GROUP) ||
0781         __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
0782         return;
0783 
0784     /* Let's now perform the deactivation */
0785     __vgic_v3_clear_active_lr(lr, lr_val);
0786 }
0787 
0788 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0789 {
0790     vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
0791 }
0792 
0793 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0794 {
0795     vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
0796 }
0797 
0798 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0799 {
0800     u64 val = vcpu_get_reg(vcpu, rt);
0801 
0802     if (val & 1)
0803         vmcr |= ICH_VMCR_ENG0_MASK;
0804     else
0805         vmcr &= ~ICH_VMCR_ENG0_MASK;
0806 
0807     __vgic_v3_write_vmcr(vmcr);
0808 }
0809 
0810 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0811 {
0812     u64 val = vcpu_get_reg(vcpu, rt);
0813 
0814     if (val & 1)
0815         vmcr |= ICH_VMCR_ENG1_MASK;
0816     else
0817         vmcr &= ~ICH_VMCR_ENG1_MASK;
0818 
0819     __vgic_v3_write_vmcr(vmcr);
0820 }
0821 
0822 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0823 {
0824     vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
0825 }
0826 
0827 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0828 {
0829     vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
0830 }
0831 
0832 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0833 {
0834     u64 val = vcpu_get_reg(vcpu, rt);
0835     u8 bpr_min = __vgic_v3_bpr_min() - 1;
0836 
0837     /* Enforce BPR limiting */
0838     if (val < bpr_min)
0839         val = bpr_min;
0840 
0841     val <<= ICH_VMCR_BPR0_SHIFT;
0842     val &= ICH_VMCR_BPR0_MASK;
0843     vmcr &= ~ICH_VMCR_BPR0_MASK;
0844     vmcr |= val;
0845 
0846     __vgic_v3_write_vmcr(vmcr);
0847 }
0848 
0849 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0850 {
0851     u64 val = vcpu_get_reg(vcpu, rt);
0852     u8 bpr_min = __vgic_v3_bpr_min();
0853 
0854     if (vmcr & ICH_VMCR_CBPR_MASK)
0855         return;
0856 
0857     /* Enforce BPR limiting */
0858     if (val < bpr_min)
0859         val = bpr_min;
0860 
0861     val <<= ICH_VMCR_BPR1_SHIFT;
0862     val &= ICH_VMCR_BPR1_MASK;
0863     vmcr &= ~ICH_VMCR_BPR1_MASK;
0864     vmcr |= val;
0865 
0866     __vgic_v3_write_vmcr(vmcr);
0867 }
0868 
0869 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
0870 {
0871     u32 val;
0872 
0873     if (!__vgic_v3_get_group(vcpu))
0874         val = __vgic_v3_read_ap0rn(n);
0875     else
0876         val = __vgic_v3_read_ap1rn(n);
0877 
0878     vcpu_set_reg(vcpu, rt, val);
0879 }
0880 
0881 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
0882 {
0883     u32 val = vcpu_get_reg(vcpu, rt);
0884 
0885     if (!__vgic_v3_get_group(vcpu))
0886         __vgic_v3_write_ap0rn(val, n);
0887     else
0888         __vgic_v3_write_ap1rn(val, n);
0889 }
0890 
0891 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
0892                         u32 vmcr, int rt)
0893 {
0894     __vgic_v3_read_apxrn(vcpu, rt, 0);
0895 }
0896 
0897 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
0898                         u32 vmcr, int rt)
0899 {
0900     __vgic_v3_read_apxrn(vcpu, rt, 1);
0901 }
0902 
0903 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0904 {
0905     __vgic_v3_read_apxrn(vcpu, rt, 2);
0906 }
0907 
0908 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0909 {
0910     __vgic_v3_read_apxrn(vcpu, rt, 3);
0911 }
0912 
0913 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0914 {
0915     __vgic_v3_write_apxrn(vcpu, rt, 0);
0916 }
0917 
0918 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0919 {
0920     __vgic_v3_write_apxrn(vcpu, rt, 1);
0921 }
0922 
0923 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0924 {
0925     __vgic_v3_write_apxrn(vcpu, rt, 2);
0926 }
0927 
0928 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0929 {
0930     __vgic_v3_write_apxrn(vcpu, rt, 3);
0931 }
0932 
0933 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0934 {
0935     u64 lr_val;
0936     int lr, lr_grp, grp;
0937 
0938     grp = __vgic_v3_get_group(vcpu);
0939 
0940     lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
0941     if (lr == -1)
0942         goto spurious;
0943 
0944     lr_grp = !!(lr_val & ICH_LR_GROUP);
0945     if (lr_grp != grp)
0946         lr_val = ICC_IAR1_EL1_SPURIOUS;
0947 
0948 spurious:
0949     vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
0950 }
0951 
0952 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0953 {
0954     vmcr &= ICH_VMCR_PMR_MASK;
0955     vmcr >>= ICH_VMCR_PMR_SHIFT;
0956     vcpu_set_reg(vcpu, rt, vmcr);
0957 }
0958 
0959 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0960 {
0961     u32 val = vcpu_get_reg(vcpu, rt);
0962 
0963     val <<= ICH_VMCR_PMR_SHIFT;
0964     val &= ICH_VMCR_PMR_MASK;
0965     vmcr &= ~ICH_VMCR_PMR_MASK;
0966     vmcr |= val;
0967 
0968     write_gicreg(vmcr, ICH_VMCR_EL2);
0969 }
0970 
0971 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0972 {
0973     u32 val = __vgic_v3_get_highest_active_priority();
0974     vcpu_set_reg(vcpu, rt, val);
0975 }
0976 
0977 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
0978 {
0979     u32 vtr, val;
0980 
0981     vtr = read_gicreg(ICH_VTR_EL2);
0982     /* PRIbits */
0983     val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
0984     /* IDbits */
0985     val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
0986     /* SEIS */
0987     if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK)
0988         val |= BIT(ICC_CTLR_EL1_SEIS_SHIFT);
0989     /* A3V */
0990     val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
0991     /* EOImode */
0992     val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
0993     /* CBPR */
0994     val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
0995 
0996     vcpu_set_reg(vcpu, rt, val);
0997 }
0998 
0999 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1000 {
1001     u32 val = vcpu_get_reg(vcpu, rt);
1002 
1003     if (val & ICC_CTLR_EL1_CBPR_MASK)
1004         vmcr |= ICH_VMCR_CBPR_MASK;
1005     else
1006         vmcr &= ~ICH_VMCR_CBPR_MASK;
1007 
1008     if (val & ICC_CTLR_EL1_EOImode_MASK)
1009         vmcr |= ICH_VMCR_EOIM_MASK;
1010     else
1011         vmcr &= ~ICH_VMCR_EOIM_MASK;
1012 
1013     write_gicreg(vmcr, ICH_VMCR_EL2);
1014 }
1015 
1016 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
1017 {
1018     int rt;
1019     u64 esr;
1020     u32 vmcr;
1021     void (*fn)(struct kvm_vcpu *, u32, int);
1022     bool is_read;
1023     u32 sysreg;
1024 
1025     esr = kvm_vcpu_get_esr(vcpu);
1026     if (vcpu_mode_is_32bit(vcpu)) {
1027         if (!kvm_condition_valid(vcpu)) {
1028             __kvm_skip_instr(vcpu);
1029             return 1;
1030         }
1031 
1032         sysreg = esr_cp15_to_sysreg(esr);
1033     } else {
1034         sysreg = esr_sys64_to_sysreg(esr);
1035     }
1036 
1037     is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
1038 
1039     switch (sysreg) {
1040     case SYS_ICC_IAR0_EL1:
1041     case SYS_ICC_IAR1_EL1:
1042         if (unlikely(!is_read))
1043             return 0;
1044         fn = __vgic_v3_read_iar;
1045         break;
1046     case SYS_ICC_EOIR0_EL1:
1047     case SYS_ICC_EOIR1_EL1:
1048         if (unlikely(is_read))
1049             return 0;
1050         fn = __vgic_v3_write_eoir;
1051         break;
1052     case SYS_ICC_IGRPEN1_EL1:
1053         if (is_read)
1054             fn = __vgic_v3_read_igrpen1;
1055         else
1056             fn = __vgic_v3_write_igrpen1;
1057         break;
1058     case SYS_ICC_BPR1_EL1:
1059         if (is_read)
1060             fn = __vgic_v3_read_bpr1;
1061         else
1062             fn = __vgic_v3_write_bpr1;
1063         break;
1064     case SYS_ICC_AP0Rn_EL1(0):
1065     case SYS_ICC_AP1Rn_EL1(0):
1066         if (is_read)
1067             fn = __vgic_v3_read_apxr0;
1068         else
1069             fn = __vgic_v3_write_apxr0;
1070         break;
1071     case SYS_ICC_AP0Rn_EL1(1):
1072     case SYS_ICC_AP1Rn_EL1(1):
1073         if (is_read)
1074             fn = __vgic_v3_read_apxr1;
1075         else
1076             fn = __vgic_v3_write_apxr1;
1077         break;
1078     case SYS_ICC_AP0Rn_EL1(2):
1079     case SYS_ICC_AP1Rn_EL1(2):
1080         if (is_read)
1081             fn = __vgic_v3_read_apxr2;
1082         else
1083             fn = __vgic_v3_write_apxr2;
1084         break;
1085     case SYS_ICC_AP0Rn_EL1(3):
1086     case SYS_ICC_AP1Rn_EL1(3):
1087         if (is_read)
1088             fn = __vgic_v3_read_apxr3;
1089         else
1090             fn = __vgic_v3_write_apxr3;
1091         break;
1092     case SYS_ICC_HPPIR0_EL1:
1093     case SYS_ICC_HPPIR1_EL1:
1094         if (unlikely(!is_read))
1095             return 0;
1096         fn = __vgic_v3_read_hppir;
1097         break;
1098     case SYS_ICC_IGRPEN0_EL1:
1099         if (is_read)
1100             fn = __vgic_v3_read_igrpen0;
1101         else
1102             fn = __vgic_v3_write_igrpen0;
1103         break;
1104     case SYS_ICC_BPR0_EL1:
1105         if (is_read)
1106             fn = __vgic_v3_read_bpr0;
1107         else
1108             fn = __vgic_v3_write_bpr0;
1109         break;
1110     case SYS_ICC_DIR_EL1:
1111         if (unlikely(is_read))
1112             return 0;
1113         fn = __vgic_v3_write_dir;
1114         break;
1115     case SYS_ICC_RPR_EL1:
1116         if (unlikely(!is_read))
1117             return 0;
1118         fn = __vgic_v3_read_rpr;
1119         break;
1120     case SYS_ICC_CTLR_EL1:
1121         if (is_read)
1122             fn = __vgic_v3_read_ctlr;
1123         else
1124             fn = __vgic_v3_write_ctlr;
1125         break;
1126     case SYS_ICC_PMR_EL1:
1127         if (is_read)
1128             fn = __vgic_v3_read_pmr;
1129         else
1130             fn = __vgic_v3_write_pmr;
1131         break;
1132     default:
1133         return 0;
1134     }
1135 
1136     vmcr = __vgic_v3_read_vmcr();
1137     rt = kvm_vcpu_sys_get_rt(vcpu);
1138     fn(vcpu, vmcr, rt);
1139 
1140     __kvm_skip_instr(vcpu);
1141 
1142     return 1;
1143 }
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!