s390/kvm/interrupt.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * handling kvm guest interrupts
0004  *
0005  * Copyright IBM Corp. 2008, 2020
0006  *
0007  *    Author(s): Carsten Otte <cotte@de.ibm.com>
0008  */
0009
0010 #define KMSG_COMPONENT "kvm-s390"
0011 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
0012
0013 #include <linux/interrupt.h>
0014 #include <linux/kvm_host.h>
0015 #include <linux/hrtimer.h>
0016 #include <linux/mmu_context.h>
0017 #include <linux/nospec.h>
0018 #include <linux/signal.h>
0019 #include <linux/slab.h>
0020 #include <linux/bitmap.h>
0021 #include <linux/vmalloc.h>
0022 #include <asm/asm-offsets.h>
0023 #include <asm/dis.h>
0024 #include <linux/uaccess.h>
0025 #include <asm/sclp.h>
0026 #include <asm/isc.h>
0027 #include <asm/gmap.h>
0028 #include <asm/switch_to.h>
0029 #include <asm/nmi.h>
0030 #include <asm/airq.h>
0031 #include <asm/tpi.h>
0032 #include "kvm-s390.h"
0033 #include "gaccess.h"
0034 #include "trace-s390.h"
0035 #include "pci.h"
0036
0037 #define PFAULT_INIT 0x0600
0038 #define PFAULT_DONE 0x0680
0039 #define VIRTIO_PARAM 0x0d00
0040
0041 static struct kvm_s390_gib *gib;
0042
0043 /* handle external calls via sigp interpretation facility */
0044 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
0045 {
0046     int c, scn;
0047
0048     if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
0049         return 0;
0050
0051     BUG_ON(!kvm_s390_use_sca_entries());
0052     read_lock(&vcpu->kvm->arch.sca_lock);
0053     if (vcpu->kvm->arch.use_esca) {
0054         struct esca_block *sca = vcpu->kvm->arch.sca;
0055         union esca_sigp_ctrl sigp_ctrl =
0056             sca->cpu[vcpu->vcpu_id].sigp_ctrl;
0057
0058         c = sigp_ctrl.c;
0059         scn = sigp_ctrl.scn;
0060     } else {
0061         struct bsca_block *sca = vcpu->kvm->arch.sca;
0062         union bsca_sigp_ctrl sigp_ctrl =
0063             sca->cpu[vcpu->vcpu_id].sigp_ctrl;
0064
0065         c = sigp_ctrl.c;
0066         scn = sigp_ctrl.scn;
0067     }
0068     read_unlock(&vcpu->kvm->arch.sca_lock);
0069
0070     if (src_id)
0071         *src_id = scn;
0072
0073     return c;
0074 }
0075
0076 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
0077 {
0078     int expect, rc;
0079
0080     BUG_ON(!kvm_s390_use_sca_entries());
0081     read_lock(&vcpu->kvm->arch.sca_lock);
0082     if (vcpu->kvm->arch.use_esca) {
0083         struct esca_block *sca = vcpu->kvm->arch.sca;
0084         union esca_sigp_ctrl *sigp_ctrl =
0085             &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
0086         union esca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl;
0087
0088         new_val.scn = src_id;
0089         new_val.c = 1;
0090         old_val.c = 0;
0091
0092         expect = old_val.value;
0093         rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
0094     } else {
0095         struct bsca_block *sca = vcpu->kvm->arch.sca;
0096         union bsca_sigp_ctrl *sigp_ctrl =
0097             &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
0098         union bsca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl;
0099
0100         new_val.scn = src_id;
0101         new_val.c = 1;
0102         old_val.c = 0;
0103
0104         expect = old_val.value;
0105         rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
0106     }
0107     read_unlock(&vcpu->kvm->arch.sca_lock);
0108
0109     if (rc != expect) {
0110         /* another external call is pending */
0111         return -EBUSY;
0112     }
0113     kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
0114     return 0;
0115 }
0116
0117 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
0118 {
0119     int rc, expect;
0120
0121     if (!kvm_s390_use_sca_entries())
0122         return;
0123     kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
0124     read_lock(&vcpu->kvm->arch.sca_lock);
0125     if (vcpu->kvm->arch.use_esca) {
0126         struct esca_block *sca = vcpu->kvm->arch.sca;
0127         union esca_sigp_ctrl *sigp_ctrl =
0128             &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
0129         union esca_sigp_ctrl old = *sigp_ctrl;
0130
0131         expect = old.value;
0132         rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
0133     } else {
0134         struct bsca_block *sca = vcpu->kvm->arch.sca;
0135         union bsca_sigp_ctrl *sigp_ctrl =
0136             &(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
0137         union bsca_sigp_ctrl old = *sigp_ctrl;
0138
0139         expect = old.value;
0140         rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
0141     }
0142     read_unlock(&vcpu->kvm->arch.sca_lock);
0143     WARN_ON(rc != expect); /* cannot clear? */
0144 }
0145
0146 int psw_extint_disabled(struct kvm_vcpu *vcpu)
0147 {
0148     return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
0149 }
0150
0151 static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
0152 {
0153     return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
0154 }
0155
0156 static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
0157 {
0158     return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
0159 }
0160
0161 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
0162 {
0163     return psw_extint_disabled(vcpu) &&
0164            psw_ioint_disabled(vcpu) &&
0165            psw_mchk_disabled(vcpu);
0166 }
0167
0168 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
0169 {
0170     if (psw_extint_disabled(vcpu) ||
0171         !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
0172         return 0;
0173     if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
0174         /* No timer interrupts when single stepping */
0175         return 0;
0176     return 1;
0177 }
0178
0179 static int ckc_irq_pending(struct kvm_vcpu *vcpu)
0180 {
0181     const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
0182     const u64 ckc = vcpu->arch.sie_block->ckc;
0183
0184     if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
0185         if ((s64)ckc >= (s64)now)
0186             return 0;
0187     } else if (ckc >= now) {
0188         return 0;
0189     }
0190     return ckc_interrupts_enabled(vcpu);
0191 }
0192
0193 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
0194 {
0195     return !psw_extint_disabled(vcpu) &&
0196            (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
0197 }
0198
0199 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
0200 {
0201     if (!cpu_timer_interrupts_enabled(vcpu))
0202         return 0;
0203     return kvm_s390_get_cpu_timer(vcpu) >> 63;
0204 }
0205
0206 static uint64_t isc_to_isc_bits(int isc)
0207 {
0208     return (0x80 >> isc) << 24;
0209 }
0210
0211 static inline u32 isc_to_int_word(u8 isc)
0212 {
0213     return ((u32)isc << 27) | 0x80000000;
0214 }
0215
0216 static inline u8 int_word_to_isc(u32 int_word)
0217 {
0218     return (int_word & 0x38000000) >> 27;
0219 }
0220
0221 /*
0222  * To use atomic bitmap functions, we have to provide a bitmap address
0223  * that is u64 aligned. However, the ipm might be u32 aligned.
0224  * Therefore, we logically start the bitmap at the very beginning of the
0225  * struct and fixup the bit number.
0226  */
0227 #define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
0228
0229 /**
0230  * gisa_set_iam - change the GISA interruption alert mask
0231  *
0232  * @gisa: gisa to operate on
0233  * @iam: new IAM value to use
0234  *
0235  * Change the IAM atomically with the next alert address and the IPM
0236  * of the GISA if the GISA is not part of the GIB alert list. All three
0237  * fields are located in the first long word of the GISA.
0238  *
0239  * Returns: 0 on success
0240  *          -EBUSY in case the gisa is part of the alert list
0241  */
0242 static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
0243 {
0244     u64 word, _word;
0245
0246     do {
0247         word = READ_ONCE(gisa->u64.word[0]);
0248         if ((u64)gisa != word >> 32)
0249             return -EBUSY;
0250         _word = (word & ~0xffUL) | iam;
0251     } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
0252
0253     return 0;
0254 }
0255
0256 /**
0257  * gisa_clear_ipm - clear the GISA interruption pending mask
0258  *
0259  * @gisa: gisa to operate on
0260  *
0261  * Clear the IPM atomically with the next alert address and the IAM
0262  * of the GISA unconditionally. All three fields are located in the
0263  * first long word of the GISA.
0264  */
0265 static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
0266 {
0267     u64 word, _word;
0268
0269     do {
0270         word = READ_ONCE(gisa->u64.word[0]);
0271         _word = word & ~(0xffUL << 24);
0272     } while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
0273 }
0274
0275 /**
0276  * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM
0277  *
0278  * @gi: gisa interrupt struct to work on
0279  *
0280  * Atomically restores the interruption alert mask if none of the
0281  * relevant ISCs are pending and return the IPM.
0282  *
0283  * Returns: the relevant pending ISCs
0284  */
0285 static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
0286 {
0287     u8 pending_mask, alert_mask;
0288     u64 word, _word;
0289
0290     do {
0291         word = READ_ONCE(gi->origin->u64.word[0]);
0292         alert_mask = READ_ONCE(gi->alert.mask);
0293         pending_mask = (u8)(word >> 24) & alert_mask;
0294         if (pending_mask)
0295             return pending_mask;
0296         _word = (word & ~0xffUL) | alert_mask;
0297     } while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word);
0298
0299     return 0;
0300 }
0301
0302 static inline int gisa_in_alert_list(struct kvm_s390_gisa *gisa)
0303 {
0304     return READ_ONCE(gisa->next_alert) != (u32)(u64)gisa;
0305 }
0306
0307 static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
0308 {
0309     set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
0310 }
0311
0312 static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa)
0313 {
0314     return READ_ONCE(gisa->ipm);
0315 }
0316
0317 static inline void gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
0318 {
0319     clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
0320 }
0321
0322 static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
0323 {
0324     return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
0325 }
0326
0327 static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
0328 {
0329     unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs |
0330                 vcpu->arch.local_int.pending_irqs;
0331
0332     pending &= ~vcpu->kvm->arch.float_int.masked_irqs;
0333     return pending;
0334 }
0335
0336 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
0337 {
0338     struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
0339     unsigned long pending_mask;
0340
0341     pending_mask = pending_irqs_no_gisa(vcpu);
0342     if (gi->origin)
0343         pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7;
0344     return pending_mask;
0345 }
0346
0347 static inline int isc_to_irq_type(unsigned long isc)
0348 {
0349     return IRQ_PEND_IO_ISC_0 - isc;
0350 }
0351
0352 static inline int irq_type_to_isc(unsigned long irq_type)
0353 {
0354     return IRQ_PEND_IO_ISC_0 - irq_type;
0355 }
0356
0357 static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
0358                    unsigned long active_mask)
0359 {
0360     int i;
0361
0362     for (i = 0; i <= MAX_ISC; i++)
0363         if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
0364             active_mask &= ~(1UL << (isc_to_irq_type(i)));
0365
0366     return active_mask;
0367 }
0368
0369 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
0370 {
0371     unsigned long active_mask;
0372
0373     active_mask = pending_irqs(vcpu);
0374     if (!active_mask)
0375         return 0;
0376
0377     if (psw_extint_disabled(vcpu))
0378         active_mask &= ~IRQ_PEND_EXT_MASK;
0379     if (psw_ioint_disabled(vcpu))
0380         active_mask &= ~IRQ_PEND_IO_MASK;
0381     else
0382         active_mask = disable_iscs(vcpu, active_mask);
0383     if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
0384         __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
0385     if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
0386         __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
0387     if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
0388         __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
0389     if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
0390         __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
0391     if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) {
0392         __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
0393         __clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask);
0394     }
0395     if (psw_mchk_disabled(vcpu))
0396         active_mask &= ~IRQ_PEND_MCHK_MASK;
0397     /* PV guest cpus can have a single interruption injected at a time. */
0398     if (kvm_s390_pv_cpu_get_handle(vcpu) &&
0399         vcpu->arch.sie_block->iictl != IICTL_CODE_NONE)
0400         active_mask &= ~(IRQ_PEND_EXT_II_MASK |
0401                  IRQ_PEND_IO_MASK |
0402                  IRQ_PEND_MCHK_MASK);
0403     /*
0404      * Check both floating and local interrupt's cr14 because
0405      * bit IRQ_PEND_MCHK_REP could be set in both cases.
0406      */
0407     if (!(vcpu->arch.sie_block->gcr[14] &
0408        (vcpu->kvm->arch.float_int.mchk.cr14 |
0409        vcpu->arch.local_int.irq.mchk.cr14)))
0410         __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
0411
0412     /*
0413      * STOP irqs will never be actively delivered. They are triggered via
0414      * intercept requests and cleared when the stop intercept is performed.
0415      */
0416     __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
0417
0418     return active_mask;
0419 }
0420
0421 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
0422 {
0423     kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
0424     set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
0425 }
0426
0427 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
0428 {
0429     kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
0430     clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
0431 }
0432
0433 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
0434 {
0435     kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
0436                       CPUSTAT_STOP_INT);
0437     vcpu->arch.sie_block->lctl = 0x0000;
0438     vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
0439
0440     if (guestdbg_enabled(vcpu)) {
0441         vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
0442                            LCTL_CR10 | LCTL_CR11);
0443         vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
0444     }
0445 }
0446
0447 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
0448 {
0449     if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
0450         return;
0451     if (psw_ioint_disabled(vcpu))
0452         kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
0453     else
0454         vcpu->arch.sie_block->lctl |= LCTL_CR6;
0455 }
0456
0457 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
0458 {
0459     if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK))
0460         return;
0461     if (psw_extint_disabled(vcpu))
0462         kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
0463     else
0464         vcpu->arch.sie_block->lctl |= LCTL_CR0;
0465 }
0466
0467 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
0468 {
0469     if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK))
0470         return;
0471     if (psw_mchk_disabled(vcpu))
0472         vcpu->arch.sie_block->ictl |= ICTL_LPSW;
0473     else
0474         vcpu->arch.sie_block->lctl |= LCTL_CR14;
0475 }
0476
0477 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
0478 {
0479     if (kvm_s390_is_stop_irq_pending(vcpu))
0480         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
0481 }
0482
0483 /* Set interception request for non-deliverable interrupts */
0484 static void set_intercept_indicators(struct kvm_vcpu *vcpu)
0485 {
0486     set_intercept_indicators_io(vcpu);
0487     set_intercept_indicators_ext(vcpu);
0488     set_intercept_indicators_mchk(vcpu);
0489     set_intercept_indicators_stop(vcpu);
0490 }
0491
0492 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
0493 {
0494     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
0495     int rc = 0;
0496
0497     vcpu->stat.deliver_cputm++;
0498     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
0499                      0, 0);
0500     if (kvm_s390_pv_cpu_is_protected(vcpu)) {
0501         vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
0502         vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER;
0503     } else {
0504         rc  = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
0505                    (u16 *)__LC_EXT_INT_CODE);
0506         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
0507         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
0508                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0509         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
0510                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0511     }
0512     clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
0513     return rc ? -EFAULT : 0;
0514 }
0515
0516 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
0517 {
0518     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
0519     int rc = 0;
0520
0521     vcpu->stat.deliver_ckc++;
0522     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
0523                      0, 0);
0524     if (kvm_s390_pv_cpu_is_protected(vcpu)) {
0525         vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
0526         vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP;
0527     } else {
0528         rc  = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
0529                    (u16 __user *)__LC_EXT_INT_CODE);
0530         rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
0531         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
0532                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0533         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
0534                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0535     }
0536     clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
0537     return rc ? -EFAULT : 0;
0538 }
0539
0540 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
0541 {
0542     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
0543     struct kvm_s390_ext_info ext;
0544     int rc;
0545
0546     spin_lock(&li->lock);
0547     ext = li->irq.ext;
0548     clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
0549     li->irq.ext.ext_params2 = 0;
0550     spin_unlock(&li->lock);
0551
0552     VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
0553            ext.ext_params2);
0554     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
0555                      KVM_S390_INT_PFAULT_INIT,
0556                      0, ext.ext_params2);
0557
0558     rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
0559     rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
0560     rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
0561                  &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0562     rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
0563                 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0564     rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
0565     return rc ? -EFAULT : 0;
0566 }
0567
0568 static int __write_machine_check(struct kvm_vcpu *vcpu,
0569                  struct kvm_s390_mchk_info *mchk)
0570 {
0571     unsigned long ext_sa_addr;
0572     unsigned long lc;
0573     freg_t fprs[NUM_FPRS];
0574     union mci mci;
0575     int rc;
0576
0577     /*
0578      * All other possible payload for a machine check (e.g. the register
0579      * contents in the save area) will be handled by the ultravisor, as
0580      * the hypervisor does not not have the needed information for
0581      * protected guests.
0582      */
0583     if (kvm_s390_pv_cpu_is_protected(vcpu)) {
0584         vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK;
0585         vcpu->arch.sie_block->mcic = mchk->mcic;
0586         vcpu->arch.sie_block->faddr = mchk->failing_storage_address;
0587         vcpu->arch.sie_block->edc = mchk->ext_damage_code;
0588         return 0;
0589     }
0590
0591     mci.val = mchk->mcic;
0592     /* take care of lazy register loading */
0593     save_fpu_regs();
0594     save_access_regs(vcpu->run->s.regs.acrs);
0595     if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
0596         save_gs_cb(current->thread.gs_cb);
0597
0598     /* Extended save area */
0599     rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
0600                sizeof(unsigned long));
0601     /* Only bits 0 through 63-LC are used for address formation */
0602     lc = ext_sa_addr & MCESA_LC_MASK;
0603     if (test_kvm_facility(vcpu->kvm, 133)) {
0604         switch (lc) {
0605         case 0:
0606         case 10:
0607             ext_sa_addr &= ~0x3ffUL;
0608             break;
0609         case 11:
0610             ext_sa_addr &= ~0x7ffUL;
0611             break;
0612         case 12:
0613             ext_sa_addr &= ~0xfffUL;
0614             break;
0615         default:
0616             ext_sa_addr = 0;
0617             break;
0618         }
0619     } else {
0620         ext_sa_addr &= ~0x3ffUL;
0621     }
0622
0623     if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
0624         if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs,
0625                     512))
0626             mci.vr = 0;
0627     } else {
0628         mci.vr = 0;
0629     }
0630     if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
0631         && (lc == 11 || lc == 12)) {
0632         if (write_guest_abs(vcpu, ext_sa_addr + 1024,
0633                     &vcpu->run->s.regs.gscb, 32))
0634             mci.gs = 0;
0635     } else {
0636         mci.gs = 0;
0637     }
0638
0639     /* General interruption information */
0640     rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
0641     rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
0642                  &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0643     rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
0644                 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0645     rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
0646
0647     /* Register-save areas */
0648     if (MACHINE_HAS_VX) {
0649         convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
0650         rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
0651     } else {
0652         rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
0653                      vcpu->run->s.regs.fprs, 128);
0654     }
0655     rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
0656                  vcpu->run->s.regs.gprs, 128);
0657     rc |= put_guest_lc(vcpu, current->thread.fpu.fpc,
0658                (u32 __user *) __LC_FP_CREG_SAVE_AREA);
0659     rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
0660                (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
0661     rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
0662                (u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
0663     rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
0664                (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
0665     rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
0666                  &vcpu->run->s.regs.acrs, 64);
0667     rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
0668                  &vcpu->arch.sie_block->gcr, 128);
0669
0670     /* Extended interruption information */
0671     rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
0672                (u32 __user *) __LC_EXT_DAMAGE_CODE);
0673     rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
0674                (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
0675     rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
0676                  sizeof(mchk->fixed_logout));
0677     return rc ? -EFAULT : 0;
0678 }
0679
0680 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
0681 {
0682     struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
0683     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
0684     struct kvm_s390_mchk_info mchk = {};
0685     int deliver = 0;
0686     int rc = 0;
0687
0688     spin_lock(&fi->lock);
0689     spin_lock(&li->lock);
0690     if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
0691         test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
0692         /*
0693          * If there was an exigent machine check pending, then any
0694          * repressible machine checks that might have been pending
0695          * are indicated along with it, so always clear bits for
0696          * repressible and exigent interrupts
0697          */
0698         mchk = li->irq.mchk;
0699         clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
0700         clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
0701         memset(&li->irq.mchk, 0, sizeof(mchk));
0702         deliver = 1;
0703     }
0704     /*
0705      * We indicate floating repressible conditions along with
0706      * other pending conditions. Channel Report Pending and Channel
0707      * Subsystem damage are the only two and are indicated by
0708      * bits in mcic and masked in cr14.
0709      */
0710     if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
0711         mchk.mcic |= fi->mchk.mcic;
0712         mchk.cr14 |= fi->mchk.cr14;
0713         memset(&fi->mchk, 0, sizeof(mchk));
0714         deliver = 1;
0715     }
0716     spin_unlock(&li->lock);
0717     spin_unlock(&fi->lock);
0718
0719     if (deliver) {
0720         VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
0721                mchk.mcic);
0722         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
0723                          KVM_S390_MCHK,
0724                          mchk.cr14, mchk.mcic);
0725         vcpu->stat.deliver_machine_check++;
0726         rc = __write_machine_check(vcpu, &mchk);
0727     }
0728     return rc;
0729 }
0730
0731 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
0732 {
0733     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
0734     int rc = 0;
0735
0736     VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
0737     vcpu->stat.deliver_restart_signal++;
0738     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
0739
0740     if (kvm_s390_pv_cpu_is_protected(vcpu)) {
0741         vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART;
0742     } else {
0743         rc  = write_guest_lc(vcpu,
0744                      offsetof(struct lowcore, restart_old_psw),
0745                      &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0746         rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
0747                     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0748     }
0749     clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
0750     return rc ? -EFAULT : 0;
0751 }
0752
0753 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
0754 {
0755     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
0756     struct kvm_s390_prefix_info prefix;
0757
0758     spin_lock(&li->lock);
0759     prefix = li->irq.prefix;
0760     li->irq.prefix.address = 0;
0761     clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
0762     spin_unlock(&li->lock);
0763
0764     vcpu->stat.deliver_prefix_signal++;
0765     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
0766                      KVM_S390_SIGP_SET_PREFIX,
0767                      prefix.address, 0);
0768
0769     kvm_s390_set_prefix(vcpu, prefix.address);
0770     return 0;
0771 }
0772
0773 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
0774 {
0775     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
0776     int rc;
0777     int cpu_addr;
0778
0779     spin_lock(&li->lock);
0780     cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
0781     clear_bit(cpu_addr, li->sigp_emerg_pending);
0782     if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
0783         clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
0784     spin_unlock(&li->lock);
0785
0786     VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
0787     vcpu->stat.deliver_emergency_signal++;
0788     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
0789                      cpu_addr, 0);
0790     if (kvm_s390_pv_cpu_is_protected(vcpu)) {
0791         vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
0792         vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG;
0793         vcpu->arch.sie_block->extcpuaddr = cpu_addr;
0794         return 0;
0795     }
0796
0797     rc  = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
0798                (u16 *)__LC_EXT_INT_CODE);
0799     rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
0800     rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
0801                  &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0802     rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
0803                 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0804     return rc ? -EFAULT : 0;
0805 }
0806
0807 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
0808 {
0809     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
0810     struct kvm_s390_extcall_info extcall;
0811     int rc;
0812
0813     spin_lock(&li->lock);
0814     extcall = li->irq.extcall;
0815     li->irq.extcall.code = 0;
0816     clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
0817     spin_unlock(&li->lock);
0818
0819     VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
0820     vcpu->stat.deliver_external_call++;
0821     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
0822                      KVM_S390_INT_EXTERNAL_CALL,
0823                      extcall.code, 0);
0824     if (kvm_s390_pv_cpu_is_protected(vcpu)) {
0825         vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
0826         vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL;
0827         vcpu->arch.sie_block->extcpuaddr = extcall.code;
0828         return 0;
0829     }
0830
0831     rc  = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
0832                (u16 *)__LC_EXT_INT_CODE);
0833     rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
0834     rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
0835                  &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0836     rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
0837                 sizeof(psw_t));
0838     return rc ? -EFAULT : 0;
0839 }
0840
0841 static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code)
0842 {
0843     switch (code) {
0844     case PGM_SPECIFICATION:
0845         vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION;
0846         break;
0847     case PGM_OPERAND:
0848         vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND;
0849         break;
0850     default:
0851         return -EINVAL;
0852     }
0853     return 0;
0854 }
0855
0856 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
0857 {
0858     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
0859     struct kvm_s390_pgm_info pgm_info;
0860     int rc = 0, nullifying = false;
0861     u16 ilen;
0862
0863     spin_lock(&li->lock);
0864     pgm_info = li->irq.pgm;
0865     clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
0866     memset(&li->irq.pgm, 0, sizeof(pgm_info));
0867     spin_unlock(&li->lock);
0868
0869     ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
0870     VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
0871            pgm_info.code, ilen);
0872     vcpu->stat.deliver_program++;
0873     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
0874                      pgm_info.code, 0);
0875
0876     /* PER is handled by the ultravisor */
0877     if (kvm_s390_pv_cpu_is_protected(vcpu))
0878         return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER);
0879
0880     switch (pgm_info.code & ~PGM_PER) {
0881     case PGM_AFX_TRANSLATION:
0882     case PGM_ASX_TRANSLATION:
0883     case PGM_EX_TRANSLATION:
0884     case PGM_LFX_TRANSLATION:
0885     case PGM_LSTE_SEQUENCE:
0886     case PGM_LSX_TRANSLATION:
0887     case PGM_LX_TRANSLATION:
0888     case PGM_PRIMARY_AUTHORITY:
0889     case PGM_SECONDARY_AUTHORITY:
0890         nullifying = true;
0891         fallthrough;
0892     case PGM_SPACE_SWITCH:
0893         rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
0894                   (u64 *)__LC_TRANS_EXC_CODE);
0895         break;
0896     case PGM_ALEN_TRANSLATION:
0897     case PGM_ALE_SEQUENCE:
0898     case PGM_ASTE_INSTANCE:
0899     case PGM_ASTE_SEQUENCE:
0900     case PGM_ASTE_VALIDITY:
0901     case PGM_EXTENDED_AUTHORITY:
0902         rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
0903                   (u8 *)__LC_EXC_ACCESS_ID);
0904         nullifying = true;
0905         break;
0906     case PGM_ASCE_TYPE:
0907     case PGM_PAGE_TRANSLATION:
0908     case PGM_REGION_FIRST_TRANS:
0909     case PGM_REGION_SECOND_TRANS:
0910     case PGM_REGION_THIRD_TRANS:
0911     case PGM_SEGMENT_TRANSLATION:
0912         rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
0913                   (u64 *)__LC_TRANS_EXC_CODE);
0914         rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
0915                    (u8 *)__LC_EXC_ACCESS_ID);
0916         rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
0917                    (u8 *)__LC_OP_ACCESS_ID);
0918         nullifying = true;
0919         break;
0920     case PGM_MONITOR:
0921         rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
0922                   (u16 *)__LC_MON_CLASS_NR);
0923         rc |= put_guest_lc(vcpu, pgm_info.mon_code,
0924                    (u64 *)__LC_MON_CODE);
0925         break;
0926     case PGM_VECTOR_PROCESSING:
0927     case PGM_DATA:
0928         rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
0929                   (u32 *)__LC_DATA_EXC_CODE);
0930         break;
0931     case PGM_PROTECTION:
0932         rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
0933                   (u64 *)__LC_TRANS_EXC_CODE);
0934         rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
0935                    (u8 *)__LC_EXC_ACCESS_ID);
0936         break;
0937     case PGM_STACK_FULL:
0938     case PGM_STACK_EMPTY:
0939     case PGM_STACK_SPECIFICATION:
0940     case PGM_STACK_TYPE:
0941     case PGM_STACK_OPERATION:
0942     case PGM_TRACE_TABEL:
0943     case PGM_CRYPTO_OPERATION:
0944         nullifying = true;
0945         break;
0946     }
0947
0948     if (pgm_info.code & PGM_PER) {
0949         rc |= put_guest_lc(vcpu, pgm_info.per_code,
0950                    (u8 *) __LC_PER_CODE);
0951         rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
0952                    (u8 *)__LC_PER_ATMID);
0953         rc |= put_guest_lc(vcpu, pgm_info.per_address,
0954                    (u64 *) __LC_PER_ADDRESS);
0955         rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
0956                    (u8 *) __LC_PER_ACCESS_ID);
0957     }
0958
0959     if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
0960         kvm_s390_rewind_psw(vcpu, ilen);
0961
0962     /* bit 1+2 of the target are the ilc, so we can directly use ilen */
0963     rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
0964     rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
0965                  (u64 *) __LC_PGM_LAST_BREAK);
0966     rc |= put_guest_lc(vcpu, pgm_info.code,
0967                (u16 *)__LC_PGM_INT_CODE);
0968     rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
0969                  &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0970     rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
0971                 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0972     return rc ? -EFAULT : 0;
0973 }
0974
0975 #define SCCB_MASK 0xFFFFFFF8
0976 #define SCCB_EVENT_PENDING 0x3
0977
0978 static int write_sclp(struct kvm_vcpu *vcpu, u32 parm)
0979 {
0980     int rc;
0981
0982     if (kvm_s390_pv_cpu_get_handle(vcpu)) {
0983         vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
0984         vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG;
0985         vcpu->arch.sie_block->eiparams = parm;
0986         return 0;
0987     }
0988
0989     rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
0990     rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
0991     rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
0992                  &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0993     rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
0994                 &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
0995     rc |= put_guest_lc(vcpu, parm,
0996                (u32 *)__LC_EXT_PARAMS);
0997
0998     return rc ? -EFAULT : 0;
0999 }
1000
1001 static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
1002 {
1003     struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1004     struct kvm_s390_ext_info ext;
1005
1006     spin_lock(&fi->lock);
1007     if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) ||
1008         !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
1009         spin_unlock(&fi->lock);
1010         return 0;
1011     }
1012     ext = fi->srv_signal;
1013     memset(&fi->srv_signal, 0, sizeof(ext));
1014     clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1015     clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1016     if (kvm_s390_pv_cpu_is_protected(vcpu))
1017         set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
1018     spin_unlock(&fi->lock);
1019
1020     VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
1021            ext.ext_params);
1022     vcpu->stat.deliver_service_signal++;
1023     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1024                      ext.ext_params, 0);
1025
1026     return write_sclp(vcpu, ext.ext_params);
1027 }
1028
1029 static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu)
1030 {
1031     struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1032     struct kvm_s390_ext_info ext;
1033
1034     spin_lock(&fi->lock);
1035     if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) {
1036         spin_unlock(&fi->lock);
1037         return 0;
1038     }
1039     ext = fi->srv_signal;
1040     /* only clear the event bit */
1041     fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING;
1042     clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1043     spin_unlock(&fi->lock);
1044
1045     VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event");
1046     vcpu->stat.deliver_service_signal++;
1047     trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
1048                      ext.ext_params, 0);
1049
1050     return write_sclp(vcpu, SCCB_EVENT_PENDING);
1051 }
1052
1053 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
1054 {
1055     struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1056     struct kvm_s390_interrupt_info *inti;
1057     int rc = 0;
1058
1059     spin_lock(&fi->lock);
1060     inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
1061                     struct kvm_s390_interrupt_info,
1062                     list);
1063     if (inti) {
1064         list_del(&inti->list);
1065         fi->counters[FIRQ_CNTR_PFAULT] -= 1;
1066     }
1067     if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
1068         clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1069     spin_unlock(&fi->lock);
1070
1071     if (inti) {
1072         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1073                          KVM_S390_INT_PFAULT_DONE, 0,
1074                          inti->ext.ext_params2);
1075         VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
1076                inti->ext.ext_params2);
1077
1078         rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1079                 (u16 *)__LC_EXT_INT_CODE);
1080         rc |= put_guest_lc(vcpu, PFAULT_DONE,
1081                 (u16 *)__LC_EXT_CPU_ADDR);
1082         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1083                 &vcpu->arch.sie_block->gpsw,
1084                 sizeof(psw_t));
1085         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1086                 &vcpu->arch.sie_block->gpsw,
1087                 sizeof(psw_t));
1088         rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1089                 (u64 *)__LC_EXT_PARAMS2);
1090         kfree(inti);
1091     }
1092     return rc ? -EFAULT : 0;
1093 }
1094
1095 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
1096 {
1097     struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
1098     struct kvm_s390_interrupt_info *inti;
1099     int rc = 0;
1100
1101     spin_lock(&fi->lock);
1102     inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
1103                     struct kvm_s390_interrupt_info,
1104                     list);
1105     if (inti) {
1106         VCPU_EVENT(vcpu, 4,
1107                "deliver: virtio parm: 0x%x,parm64: 0x%llx",
1108                inti->ext.ext_params, inti->ext.ext_params2);
1109         vcpu->stat.deliver_virtio++;
1110         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1111                 inti->type,
1112                 inti->ext.ext_params,
1113                 inti->ext.ext_params2);
1114         list_del(&inti->list);
1115         fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
1116     }
1117     if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
1118         clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1119     spin_unlock(&fi->lock);
1120
1121     if (inti) {
1122         rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
1123                 (u16 *)__LC_EXT_INT_CODE);
1124         rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
1125                 (u16 *)__LC_EXT_CPU_ADDR);
1126         rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
1127                 &vcpu->arch.sie_block->gpsw,
1128                 sizeof(psw_t));
1129         rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
1130                 &vcpu->arch.sie_block->gpsw,
1131                 sizeof(psw_t));
1132         rc |= put_guest_lc(vcpu, inti->ext.ext_params,
1133                 (u32 *)__LC_EXT_PARAMS);
1134         rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
1135                 (u64 *)__LC_EXT_PARAMS2);
1136         kfree(inti);
1137     }
1138     return rc ? -EFAULT : 0;
1139 }
1140
1141 static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
1142 {
1143     int rc;
1144
1145     if (kvm_s390_pv_cpu_is_protected(vcpu)) {
1146         vcpu->arch.sie_block->iictl = IICTL_CODE_IO;
1147         vcpu->arch.sie_block->subchannel_id = io->subchannel_id;
1148         vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr;
1149         vcpu->arch.sie_block->io_int_parm = io->io_int_parm;
1150         vcpu->arch.sie_block->io_int_word = io->io_int_word;
1151         return 0;
1152     }
1153
1154     rc  = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
1155     rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
1156     rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
1157     rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
1158     rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
1159                  &vcpu->arch.sie_block->gpsw,
1160                  sizeof(psw_t));
1161     rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
1162                 &vcpu->arch.sie_block->gpsw,
1163                 sizeof(psw_t));
1164     return rc ? -EFAULT : 0;
1165 }
1166
1167 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
1168                      unsigned long irq_type)
1169 {
1170     struct list_head *isc_list;
1171     struct kvm_s390_float_interrupt *fi;
1172     struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1173     struct kvm_s390_interrupt_info *inti = NULL;
1174     struct kvm_s390_io_info io;
1175     u32 isc;
1176     int rc = 0;
1177
1178     fi = &vcpu->kvm->arch.float_int;
1179
1180     spin_lock(&fi->lock);
1181     isc = irq_type_to_isc(irq_type);
1182     isc_list = &fi->lists[isc];
1183     inti = list_first_entry_or_null(isc_list,
1184                     struct kvm_s390_interrupt_info,
1185                     list);
1186     if (inti) {
1187         if (inti->type & KVM_S390_INT_IO_AI_MASK)
1188             VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
1189         else
1190             VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
1191             inti->io.subchannel_id >> 8,
1192             inti->io.subchannel_id >> 1 & 0x3,
1193             inti->io.subchannel_nr);
1194
1195         vcpu->stat.deliver_io++;
1196         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1197                 inti->type,
1198                 ((__u32)inti->io.subchannel_id << 16) |
1199                 inti->io.subchannel_nr,
1200                 ((__u64)inti->io.io_int_parm << 32) |
1201                 inti->io.io_int_word);
1202         list_del(&inti->list);
1203         fi->counters[FIRQ_CNTR_IO] -= 1;
1204     }
1205     if (list_empty(isc_list))
1206         clear_bit(irq_type, &fi->pending_irqs);
1207     spin_unlock(&fi->lock);
1208
1209     if (inti) {
1210         rc = __do_deliver_io(vcpu, &(inti->io));
1211         kfree(inti);
1212         goto out;
1213     }
1214
1215     if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) {
1216         /*
1217          * in case an adapter interrupt was not delivered
1218          * in SIE context KVM will handle the delivery
1219          */
1220         VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
1221         memset(&io, 0, sizeof(io));
1222         io.io_int_word = isc_to_int_word(isc);
1223         vcpu->stat.deliver_io++;
1224         trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
1225             KVM_S390_INT_IO(1, 0, 0, 0),
1226             ((__u32)io.subchannel_id << 16) |
1227             io.subchannel_nr,
1228             ((__u64)io.io_int_parm << 32) |
1229             io.io_int_word);
1230         rc = __do_deliver_io(vcpu, &io);
1231     }
1232 out:
1233     return rc;
1234 }
1235
1236 /* Check whether an external call is pending (deliverable or not) */
1237 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
1238 {
1239     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1240
1241     if (!sclp.has_sigpif)
1242         return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
1243
1244     return sca_ext_call_pending(vcpu, NULL);
1245 }
1246
1247 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
1248 {
1249     if (deliverable_irqs(vcpu))
1250         return 1;
1251
1252     if (kvm_cpu_has_pending_timer(vcpu))
1253         return 1;
1254
1255     /* external call pending and deliverable */
1256     if (kvm_s390_ext_call_pending(vcpu) &&
1257         !psw_extint_disabled(vcpu) &&
1258         (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
1259         return 1;
1260
1261     if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
1262         return 1;
1263     return 0;
1264 }
1265
1266 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
1267 {
1268     return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
1269 }
1270
1271 static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
1272 {
1273     const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
1274     const u64 ckc = vcpu->arch.sie_block->ckc;
1275     u64 cputm, sltime = 0;
1276
1277     if (ckc_interrupts_enabled(vcpu)) {
1278         if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
1279             if ((s64)now < (s64)ckc)
1280                 sltime = tod_to_ns((s64)ckc - (s64)now);
1281         } else if (now < ckc) {
1282             sltime = tod_to_ns(ckc - now);
1283         }
1284         /* already expired */
1285         if (!sltime)
1286             return 0;
1287         if (cpu_timer_interrupts_enabled(vcpu)) {
1288             cputm = kvm_s390_get_cpu_timer(vcpu);
1289             /* already expired? */
1290             if (cputm >> 63)
1291                 return 0;
1292             return min_t(u64, sltime, tod_to_ns(cputm));
1293         }
1294     } else if (cpu_timer_interrupts_enabled(vcpu)) {
1295         sltime = kvm_s390_get_cpu_timer(vcpu);
1296         /* already expired? */
1297         if (sltime >> 63)
1298             return 0;
1299     }
1300     return sltime;
1301 }
1302
1303 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
1304 {
1305     struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
1306     u64 sltime;
1307
1308     vcpu->stat.exit_wait_state++;
1309
1310     /* fast path */
1311     if (kvm_arch_vcpu_runnable(vcpu))
1312         return 0;
1313
1314     if (psw_interrupts_disabled(vcpu)) {
1315         VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
1316         return -EOPNOTSUPP; /* disabled wait */
1317     }
1318
1319     if (gi->origin &&
1320         (gisa_get_ipm_or_restore_iam(gi) &
1321          vcpu->arch.sie_block->gcr[6] >> 24))
1322         return 0;
1323
1324     if (!ckc_interrupts_enabled(vcpu) &&
1325         !cpu_timer_interrupts_enabled(vcpu)) {
1326         VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
1327         __set_cpu_idle(vcpu);
1328         goto no_timer;
1329     }
1330
1331     sltime = __calculate_sltime(vcpu);
1332     if (!sltime)
1333         return 0;
1334
1335     __set_cpu_idle(vcpu);
1336     hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL);
1337     VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
1338 no_timer:
1339     kvm_vcpu_srcu_read_unlock(vcpu);
1340     kvm_vcpu_halt(vcpu);
1341     vcpu->valid_wakeup = false;
1342     __unset_cpu_idle(vcpu);
1343     kvm_vcpu_srcu_read_lock(vcpu);
1344
1345     hrtimer_cancel(&vcpu->arch.ckc_timer);
1346     return 0;
1347 }
1348
1349 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
1350 {
1351     vcpu->valid_wakeup = true;
1352     kvm_vcpu_wake_up(vcpu);
1353
1354     /*
1355      * The VCPU might not be sleeping but rather executing VSIE. Let's
1356      * kick it, so it leaves the SIE to process the request.
1357      */
1358     kvm_s390_vsie_kick(vcpu);
1359 }
1360
1361 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
1362 {
1363     struct kvm_vcpu *vcpu;
1364     u64 sltime;
1365
1366     vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
1367     sltime = __calculate_sltime(vcpu);
1368
1369     /*
1370      * If the monotonic clock runs faster than the tod clock we might be
1371      * woken up too early and have to go back to sleep to avoid deadlocks.
1372      */
1373     if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime)))
1374         return HRTIMER_RESTART;
1375     kvm_s390_vcpu_wakeup(vcpu);
1376     return HRTIMER_NORESTART;
1377 }
1378
1379 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
1380 {
1381     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1382
1383     spin_lock(&li->lock);
1384     li->pending_irqs = 0;
1385     bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
1386     memset(&li->irq, 0, sizeof(li->irq));
1387     spin_unlock(&li->lock);
1388
1389     sca_clear_ext_call(vcpu);
1390 }
1391
1392 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
1393 {
1394     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1395     int rc = 0;
1396     unsigned long irq_type;
1397     unsigned long irqs;
1398
1399     __reset_intercept_indicators(vcpu);
1400
1401     /* pending ckc conditions might have been invalidated */
1402     clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1403     if (ckc_irq_pending(vcpu))
1404         set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1405
1406     /* pending cpu timer conditions might have been invalidated */
1407     clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1408     if (cpu_timer_irq_pending(vcpu))
1409         set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1410
1411     while ((irqs = deliverable_irqs(vcpu)) && !rc) {
1412         /* bits are in the reverse order of interrupt priority */
1413         irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
1414         switch (irq_type) {
1415         case IRQ_PEND_IO_ISC_0:
1416         case IRQ_PEND_IO_ISC_1:
1417         case IRQ_PEND_IO_ISC_2:
1418         case IRQ_PEND_IO_ISC_3:
1419         case IRQ_PEND_IO_ISC_4:
1420         case IRQ_PEND_IO_ISC_5:
1421         case IRQ_PEND_IO_ISC_6:
1422         case IRQ_PEND_IO_ISC_7:
1423             rc = __deliver_io(vcpu, irq_type);
1424             break;
1425         case IRQ_PEND_MCHK_EX:
1426         case IRQ_PEND_MCHK_REP:
1427             rc = __deliver_machine_check(vcpu);
1428             break;
1429         case IRQ_PEND_PROG:
1430             rc = __deliver_prog(vcpu);
1431             break;
1432         case IRQ_PEND_EXT_EMERGENCY:
1433             rc = __deliver_emergency_signal(vcpu);
1434             break;
1435         case IRQ_PEND_EXT_EXTERNAL:
1436             rc = __deliver_external_call(vcpu);
1437             break;
1438         case IRQ_PEND_EXT_CLOCK_COMP:
1439             rc = __deliver_ckc(vcpu);
1440             break;
1441         case IRQ_PEND_EXT_CPU_TIMER:
1442             rc = __deliver_cpu_timer(vcpu);
1443             break;
1444         case IRQ_PEND_RESTART:
1445             rc = __deliver_restart(vcpu);
1446             break;
1447         case IRQ_PEND_SET_PREFIX:
1448             rc = __deliver_set_prefix(vcpu);
1449             break;
1450         case IRQ_PEND_PFAULT_INIT:
1451             rc = __deliver_pfault_init(vcpu);
1452             break;
1453         case IRQ_PEND_EXT_SERVICE:
1454             rc = __deliver_service(vcpu);
1455             break;
1456         case IRQ_PEND_EXT_SERVICE_EV:
1457             rc = __deliver_service_ev(vcpu);
1458             break;
1459         case IRQ_PEND_PFAULT_DONE:
1460             rc = __deliver_pfault_done(vcpu);
1461             break;
1462         case IRQ_PEND_VIRTIO:
1463             rc = __deliver_virtio(vcpu);
1464             break;
1465         default:
1466             WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
1467             clear_bit(irq_type, &li->pending_irqs);
1468         }
1469     }
1470
1471     set_intercept_indicators(vcpu);
1472
1473     return rc;
1474 }
1475
1476 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1477 {
1478     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1479
1480     vcpu->stat.inject_program++;
1481     VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
1482     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
1483                    irq->u.pgm.code, 0);
1484
1485     if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
1486         /* auto detection if no valid ILC was given */
1487         irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
1488         irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
1489         irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
1490     }
1491
1492     if (irq->u.pgm.code == PGM_PER) {
1493         li->irq.pgm.code |= PGM_PER;
1494         li->irq.pgm.flags = irq->u.pgm.flags;
1495         /* only modify PER related information */
1496         li->irq.pgm.per_address = irq->u.pgm.per_address;
1497         li->irq.pgm.per_code = irq->u.pgm.per_code;
1498         li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
1499         li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
1500     } else if (!(irq->u.pgm.code & PGM_PER)) {
1501         li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
1502                    irq->u.pgm.code;
1503         li->irq.pgm.flags = irq->u.pgm.flags;
1504         /* only modify non-PER information */
1505         li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
1506         li->irq.pgm.mon_code = irq->u.pgm.mon_code;
1507         li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
1508         li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
1509         li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
1510         li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
1511     } else {
1512         li->irq.pgm = irq->u.pgm;
1513     }
1514     set_bit(IRQ_PEND_PROG, &li->pending_irqs);
1515     return 0;
1516 }
1517
1518 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1519 {
1520     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1521
1522     vcpu->stat.inject_pfault_init++;
1523     VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
1524            irq->u.ext.ext_params2);
1525     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
1526                    irq->u.ext.ext_params,
1527                    irq->u.ext.ext_params2);
1528
1529     li->irq.ext = irq->u.ext;
1530     set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
1531     kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1532     return 0;
1533 }
1534
1535 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1536 {
1537     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1538     struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
1539     uint16_t src_id = irq->u.extcall.code;
1540
1541     vcpu->stat.inject_external_call++;
1542     VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
1543            src_id);
1544     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
1545                    src_id, 0);
1546
1547     /* sending vcpu invalid */
1548     if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
1549         return -EINVAL;
1550
1551     if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu))
1552         return sca_inject_ext_call(vcpu, src_id);
1553
1554     if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
1555         return -EBUSY;
1556     *extcall = irq->u.extcall;
1557     kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1558     return 0;
1559 }
1560
1561 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1562 {
1563     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1564     struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
1565
1566     vcpu->stat.inject_set_prefix++;
1567     VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
1568            irq->u.prefix.address);
1569     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
1570                    irq->u.prefix.address, 0);
1571
1572     if (!is_vcpu_stopped(vcpu))
1573         return -EBUSY;
1574
1575     *prefix = irq->u.prefix;
1576     set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
1577     return 0;
1578 }
1579
1580 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
1581 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1582 {
1583     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1584     struct kvm_s390_stop_info *stop = &li->irq.stop;
1585     int rc = 0;
1586
1587     vcpu->stat.inject_stop_signal++;
1588     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
1589
1590     if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
1591         return -EINVAL;
1592
1593     if (is_vcpu_stopped(vcpu)) {
1594         if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
1595             rc = kvm_s390_store_status_unloaded(vcpu,
1596                         KVM_S390_STORE_STATUS_NOADDR);
1597         return rc;
1598     }
1599
1600     if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
1601         return -EBUSY;
1602     stop->flags = irq->u.stop.flags;
1603     kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
1604     return 0;
1605 }
1606
1607 static int __inject_sigp_restart(struct kvm_vcpu *vcpu)
1608 {
1609     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1610
1611     vcpu->stat.inject_restart++;
1612     VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
1613     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
1614
1615     set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
1616     return 0;
1617 }
1618
1619 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
1620                    struct kvm_s390_irq *irq)
1621 {
1622     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1623
1624     vcpu->stat.inject_emergency_signal++;
1625     VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
1626            irq->u.emerg.code);
1627     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
1628                    irq->u.emerg.code, 0);
1629
1630     /* sending vcpu invalid */
1631     if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
1632         return -EINVAL;
1633
1634     set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
1635     set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
1636     kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1637     return 0;
1638 }
1639
1640 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
1641 {
1642     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1643     struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
1644
1645     vcpu->stat.inject_mchk++;
1646     VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
1647            irq->u.mchk.mcic);
1648     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
1649                    irq->u.mchk.mcic);
1650
1651     /*
1652      * Because repressible machine checks can be indicated along with
1653      * exigent machine checks (PoP, Chapter 11, Interruption action)
1654      * we need to combine cr14, mcic and external damage code.
1655      * Failing storage address and the logout area should not be or'ed
1656      * together, we just indicate the last occurrence of the corresponding
1657      * machine check
1658      */
1659     mchk->cr14 |= irq->u.mchk.cr14;
1660     mchk->mcic |= irq->u.mchk.mcic;
1661     mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
1662     mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
1663     memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
1664            sizeof(mchk->fixed_logout));
1665     if (mchk->mcic & MCHK_EX_MASK)
1666         set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
1667     else if (mchk->mcic & MCHK_REP_MASK)
1668         set_bit(IRQ_PEND_MCHK_REP,  &li->pending_irqs);
1669     return 0;
1670 }
1671
1672 static int __inject_ckc(struct kvm_vcpu *vcpu)
1673 {
1674     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1675
1676     vcpu->stat.inject_ckc++;
1677     VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
1678     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
1679                    0, 0);
1680
1681     set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
1682     kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1683     return 0;
1684 }
1685
1686 static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
1687 {
1688     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
1689
1690     vcpu->stat.inject_cputm++;
1691     VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
1692     trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
1693                    0, 0);
1694
1695     set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
1696     kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
1697     return 0;
1698 }
1699
1700 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
1701                           int isc, u32 schid)
1702 {
1703     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1704     struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1705     struct kvm_s390_interrupt_info *iter;
1706     u16 id = (schid & 0xffff0000U) >> 16;
1707     u16 nr = schid & 0x0000ffffU;
1708
1709     spin_lock(&fi->lock);
1710     list_for_each_entry(iter, isc_list, list) {
1711         if (schid && (id != iter->io.subchannel_id ||
1712                   nr != iter->io.subchannel_nr))
1713             continue;
1714         /* found an appropriate entry */
1715         list_del_init(&iter->list);
1716         fi->counters[FIRQ_CNTR_IO] -= 1;
1717         if (list_empty(isc_list))
1718             clear_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1719         spin_unlock(&fi->lock);
1720         return iter;
1721     }
1722     spin_unlock(&fi->lock);
1723     return NULL;
1724 }
1725
1726 static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
1727                               u64 isc_mask, u32 schid)
1728 {
1729     struct kvm_s390_interrupt_info *inti = NULL;
1730     int isc;
1731
1732     for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
1733         if (isc_mask & isc_to_isc_bits(isc))
1734             inti = get_io_int(kvm, isc, schid);
1735     }
1736     return inti;
1737 }
1738
1739 static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
1740 {
1741     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1742     unsigned long active_mask;
1743     int isc;
1744
1745     if (schid)
1746         goto out;
1747     if (!gi->origin)
1748         goto out;
1749
1750     active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32;
1751     while (active_mask) {
1752         isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
1753         if (gisa_tac_ipm_gisc(gi->origin, isc))
1754             return isc;
1755         clear_bit_inv(isc, &active_mask);
1756     }
1757 out:
1758     return -EINVAL;
1759 }
1760
1761 /*
1762  * Dequeue and return an I/O interrupt matching any of the interruption
1763  * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
1764  * Take into account the interrupts pending in the interrupt list and in GISA.
1765  *
1766  * Note that for a guest that does not enable I/O interrupts
1767  * but relies on TPI, a flood of classic interrupts may starve
1768  * out adapter interrupts on the same isc. Linux does not do
1769  * that, and it is possible to work around the issue by configuring
1770  * different iscs for classic and adapter interrupts in the guest,
1771  * but we may want to revisit this in the future.
1772  */
1773 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
1774                             u64 isc_mask, u32 schid)
1775 {
1776     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1777     struct kvm_s390_interrupt_info *inti, *tmp_inti;
1778     int isc;
1779
1780     inti = get_top_io_int(kvm, isc_mask, schid);
1781
1782     isc = get_top_gisa_isc(kvm, isc_mask, schid);
1783     if (isc < 0)
1784         /* no AI in GISA */
1785         goto out;
1786
1787     if (!inti)
1788         /* AI in GISA but no classical IO int */
1789         goto gisa_out;
1790
1791     /* both types of interrupts present */
1792     if (int_word_to_isc(inti->io.io_int_word) <= isc) {
1793         /* classical IO int with higher priority */
1794         gisa_set_ipm_gisc(gi->origin, isc);
1795         goto out;
1796     }
1797 gisa_out:
1798     tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
1799     if (tmp_inti) {
1800         tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
1801         tmp_inti->io.io_int_word = isc_to_int_word(isc);
1802         if (inti)
1803             kvm_s390_reinject_io_int(kvm, inti);
1804         inti = tmp_inti;
1805     } else
1806         gisa_set_ipm_gisc(gi->origin, isc);
1807 out:
1808     return inti;
1809 }
1810
1811 static int __inject_service(struct kvm *kvm,
1812                  struct kvm_s390_interrupt_info *inti)
1813 {
1814     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1815
1816     kvm->stat.inject_service_signal++;
1817     spin_lock(&fi->lock);
1818     fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
1819
1820     /* We always allow events, track them separately from the sccb ints */
1821     if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING)
1822         set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
1823
1824     /*
1825      * Early versions of the QEMU s390 bios will inject several
1826      * service interrupts after another without handling a
1827      * condition code indicating busy.
1828      * We will silently ignore those superfluous sccb values.
1829      * A future version of QEMU will take care of serialization
1830      * of servc requests
1831      */
1832     if (fi->srv_signal.ext_params & SCCB_MASK)
1833         goto out;
1834     fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
1835     set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
1836 out:
1837     spin_unlock(&fi->lock);
1838     kfree(inti);
1839     return 0;
1840 }
1841
1842 static int __inject_virtio(struct kvm *kvm,
1843                 struct kvm_s390_interrupt_info *inti)
1844 {
1845     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1846
1847     kvm->stat.inject_virtio++;
1848     spin_lock(&fi->lock);
1849     if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
1850         spin_unlock(&fi->lock);
1851         return -EBUSY;
1852     }
1853     fi->counters[FIRQ_CNTR_VIRTIO] += 1;
1854     list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
1855     set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
1856     spin_unlock(&fi->lock);
1857     return 0;
1858 }
1859
1860 static int __inject_pfault_done(struct kvm *kvm,
1861                  struct kvm_s390_interrupt_info *inti)
1862 {
1863     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1864
1865     kvm->stat.inject_pfault_done++;
1866     spin_lock(&fi->lock);
1867     if (fi->counters[FIRQ_CNTR_PFAULT] >=
1868         (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
1869         spin_unlock(&fi->lock);
1870         return -EBUSY;
1871     }
1872     fi->counters[FIRQ_CNTR_PFAULT] += 1;
1873     list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
1874     set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
1875     spin_unlock(&fi->lock);
1876     return 0;
1877 }
1878
1879 #define CR_PENDING_SUBCLASS 28
1880 static int __inject_float_mchk(struct kvm *kvm,
1881                 struct kvm_s390_interrupt_info *inti)
1882 {
1883     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
1884
1885     kvm->stat.inject_float_mchk++;
1886     spin_lock(&fi->lock);
1887     fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
1888     fi->mchk.mcic |= inti->mchk.mcic;
1889     set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
1890     spin_unlock(&fi->lock);
1891     kfree(inti);
1892     return 0;
1893 }
1894
1895 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1896 {
1897     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
1898     struct kvm_s390_float_interrupt *fi;
1899     struct list_head *list;
1900     int isc;
1901
1902     kvm->stat.inject_io++;
1903     isc = int_word_to_isc(inti->io.io_int_word);
1904
1905     /*
1906      * We do not use the lock checking variant as this is just a
1907      * performance optimization and we do not hold the lock here.
1908      * This is ok as the code will pick interrupts from both "lists"
1909      * for delivery.
1910      */
1911     if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) {
1912         VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
1913         gisa_set_ipm_gisc(gi->origin, isc);
1914         kfree(inti);
1915         return 0;
1916     }
1917
1918     fi = &kvm->arch.float_int;
1919     spin_lock(&fi->lock);
1920     if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
1921         spin_unlock(&fi->lock);
1922         return -EBUSY;
1923     }
1924     fi->counters[FIRQ_CNTR_IO] += 1;
1925
1926     if (inti->type & KVM_S390_INT_IO_AI_MASK)
1927         VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
1928     else
1929         VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
1930             inti->io.subchannel_id >> 8,
1931             inti->io.subchannel_id >> 1 & 0x3,
1932             inti->io.subchannel_nr);
1933     list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
1934     list_add_tail(&inti->list, list);
1935     set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
1936     spin_unlock(&fi->lock);
1937     return 0;
1938 }
1939
1940 /*
1941  * Find a destination VCPU for a floating irq and kick it.
1942  */
1943 static void __floating_irq_kick(struct kvm *kvm, u64 type)
1944 {
1945     struct kvm_vcpu *dst_vcpu;
1946     int sigcpu, online_vcpus, nr_tries = 0;
1947
1948     online_vcpus = atomic_read(&kvm->online_vcpus);
1949     if (!online_vcpus)
1950         return;
1951
1952     /* find idle VCPUs first, then round robin */
1953     sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus);
1954     if (sigcpu == online_vcpus) {
1955         do {
1956             sigcpu = kvm->arch.float_int.next_rr_cpu++;
1957             kvm->arch.float_int.next_rr_cpu %= online_vcpus;
1958             /* avoid endless loops if all vcpus are stopped */
1959             if (nr_tries++ >= online_vcpus)
1960                 return;
1961         } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
1962     }
1963     dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
1964
1965     /* make the VCPU drop out of the SIE, or wake it up if sleeping */
1966     switch (type) {
1967     case KVM_S390_MCHK:
1968         kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
1969         break;
1970     case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
1971         if (!(type & KVM_S390_INT_IO_AI_MASK &&
1972               kvm->arch.gisa_int.origin) ||
1973               kvm_s390_pv_cpu_get_handle(dst_vcpu))
1974             kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
1975         break;
1976     default:
1977         kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
1978         break;
1979     }
1980     kvm_s390_vcpu_wakeup(dst_vcpu);
1981 }
1982
1983 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
1984 {
1985     u64 type = READ_ONCE(inti->type);
1986     int rc;
1987
1988     switch (type) {
1989     case KVM_S390_MCHK:
1990         rc = __inject_float_mchk(kvm, inti);
1991         break;
1992     case KVM_S390_INT_VIRTIO:
1993         rc = __inject_virtio(kvm, inti);
1994         break;
1995     case KVM_S390_INT_SERVICE:
1996         rc = __inject_service(kvm, inti);
1997         break;
1998     case KVM_S390_INT_PFAULT_DONE:
1999         rc = __inject_pfault_done(kvm, inti);
2000         break;
2001     case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2002         rc = __inject_io(kvm, inti);
2003         break;
2004     default:
2005         rc = -EINVAL;
2006     }
2007     if (rc)
2008         return rc;
2009
2010     __floating_irq_kick(kvm, type);
2011     return 0;
2012 }
2013
2014 int kvm_s390_inject_vm(struct kvm *kvm,
2015                struct kvm_s390_interrupt *s390int)
2016 {
2017     struct kvm_s390_interrupt_info *inti;
2018     int rc;
2019
2020     inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2021     if (!inti)
2022         return -ENOMEM;
2023
2024     inti->type = s390int->type;
2025     switch (inti->type) {
2026     case KVM_S390_INT_VIRTIO:
2027         VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
2028              s390int->parm, s390int->parm64);
2029         inti->ext.ext_params = s390int->parm;
2030         inti->ext.ext_params2 = s390int->parm64;
2031         break;
2032     case KVM_S390_INT_SERVICE:
2033         VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
2034         inti->ext.ext_params = s390int->parm;
2035         break;
2036     case KVM_S390_INT_PFAULT_DONE:
2037         inti->ext.ext_params2 = s390int->parm64;
2038         break;
2039     case KVM_S390_MCHK:
2040         VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
2041              s390int->parm64);
2042         inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
2043         inti->mchk.mcic = s390int->parm64;
2044         break;
2045     case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2046         inti->io.subchannel_id = s390int->parm >> 16;
2047         inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
2048         inti->io.io_int_parm = s390int->parm64 >> 32;
2049         inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
2050         break;
2051     default:
2052         kfree(inti);
2053         return -EINVAL;
2054     }
2055     trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
2056                  2);
2057
2058     rc = __inject_vm(kvm, inti);
2059     if (rc)
2060         kfree(inti);
2061     return rc;
2062 }
2063
2064 int kvm_s390_reinject_io_int(struct kvm *kvm,
2065                   struct kvm_s390_interrupt_info *inti)
2066 {
2067     return __inject_vm(kvm, inti);
2068 }
2069
2070 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
2071                struct kvm_s390_irq *irq)
2072 {
2073     irq->type = s390int->type;
2074     switch (irq->type) {
2075     case KVM_S390_PROGRAM_INT:
2076         if (s390int->parm & 0xffff0000)
2077             return -EINVAL;
2078         irq->u.pgm.code = s390int->parm;
2079         break;
2080     case KVM_S390_SIGP_SET_PREFIX:
2081         irq->u.prefix.address = s390int->parm;
2082         break;
2083     case KVM_S390_SIGP_STOP:
2084         irq->u.stop.flags = s390int->parm;
2085         break;
2086     case KVM_S390_INT_EXTERNAL_CALL:
2087         if (s390int->parm & 0xffff0000)
2088             return -EINVAL;
2089         irq->u.extcall.code = s390int->parm;
2090         break;
2091     case KVM_S390_INT_EMERGENCY:
2092         if (s390int->parm & 0xffff0000)
2093             return -EINVAL;
2094         irq->u.emerg.code = s390int->parm;
2095         break;
2096     case KVM_S390_MCHK:
2097         irq->u.mchk.mcic = s390int->parm64;
2098         break;
2099     case KVM_S390_INT_PFAULT_INIT:
2100         irq->u.ext.ext_params = s390int->parm;
2101         irq->u.ext.ext_params2 = s390int->parm64;
2102         break;
2103     case KVM_S390_RESTART:
2104     case KVM_S390_INT_CLOCK_COMP:
2105     case KVM_S390_INT_CPU_TIMER:
2106         break;
2107     default:
2108         return -EINVAL;
2109     }
2110     return 0;
2111 }
2112
2113 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
2114 {
2115     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2116
2117     return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2118 }
2119
2120 int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
2121 {
2122     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2123
2124     return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
2125 }
2126
2127 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
2128 {
2129     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2130
2131     spin_lock(&li->lock);
2132     li->irq.stop.flags = 0;
2133     clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
2134     spin_unlock(&li->lock);
2135 }
2136
2137 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2138 {
2139     int rc;
2140
2141     switch (irq->type) {
2142     case KVM_S390_PROGRAM_INT:
2143         rc = __inject_prog(vcpu, irq);
2144         break;
2145     case KVM_S390_SIGP_SET_PREFIX:
2146         rc = __inject_set_prefix(vcpu, irq);
2147         break;
2148     case KVM_S390_SIGP_STOP:
2149         rc = __inject_sigp_stop(vcpu, irq);
2150         break;
2151     case KVM_S390_RESTART:
2152         rc = __inject_sigp_restart(vcpu);
2153         break;
2154     case KVM_S390_INT_CLOCK_COMP:
2155         rc = __inject_ckc(vcpu);
2156         break;
2157     case KVM_S390_INT_CPU_TIMER:
2158         rc = __inject_cpu_timer(vcpu);
2159         break;
2160     case KVM_S390_INT_EXTERNAL_CALL:
2161         rc = __inject_extcall(vcpu, irq);
2162         break;
2163     case KVM_S390_INT_EMERGENCY:
2164         rc = __inject_sigp_emergency(vcpu, irq);
2165         break;
2166     case KVM_S390_MCHK:
2167         rc = __inject_mchk(vcpu, irq);
2168         break;
2169     case KVM_S390_INT_PFAULT_INIT:
2170         rc = __inject_pfault_init(vcpu, irq);
2171         break;
2172     case KVM_S390_INT_VIRTIO:
2173     case KVM_S390_INT_SERVICE:
2174     case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2175     default:
2176         rc = -EINVAL;
2177     }
2178
2179     return rc;
2180 }
2181
2182 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
2183 {
2184     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2185     int rc;
2186
2187     spin_lock(&li->lock);
2188     rc = do_inject_vcpu(vcpu, irq);
2189     spin_unlock(&li->lock);
2190     if (!rc)
2191         kvm_s390_vcpu_wakeup(vcpu);
2192     return rc;
2193 }
2194
2195 static inline void clear_irq_list(struct list_head *_list)
2196 {
2197     struct kvm_s390_interrupt_info *inti, *n;
2198
2199     list_for_each_entry_safe(inti, n, _list, list) {
2200         list_del(&inti->list);
2201         kfree(inti);
2202     }
2203 }
2204
2205 static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
2206                struct kvm_s390_irq *irq)
2207 {
2208     irq->type = inti->type;
2209     switch (inti->type) {
2210     case KVM_S390_INT_PFAULT_INIT:
2211     case KVM_S390_INT_PFAULT_DONE:
2212     case KVM_S390_INT_VIRTIO:
2213         irq->u.ext = inti->ext;
2214         break;
2215     case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2216         irq->u.io = inti->io;
2217         break;
2218     }
2219 }
2220
2221 void kvm_s390_clear_float_irqs(struct kvm *kvm)
2222 {
2223     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2224     int i;
2225
2226     mutex_lock(&kvm->lock);
2227     if (!kvm_s390_pv_is_protected(kvm))
2228         fi->masked_irqs = 0;
2229     mutex_unlock(&kvm->lock);
2230     spin_lock(&fi->lock);
2231     fi->pending_irqs = 0;
2232     memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
2233     memset(&fi->mchk, 0, sizeof(fi->mchk));
2234     for (i = 0; i < FIRQ_LIST_COUNT; i++)
2235         clear_irq_list(&fi->lists[i]);
2236     for (i = 0; i < FIRQ_MAX_COUNT; i++)
2237         fi->counters[i] = 0;
2238     spin_unlock(&fi->lock);
2239     kvm_s390_gisa_clear(kvm);
2240 };
2241
2242 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
2243 {
2244     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
2245     struct kvm_s390_interrupt_info *inti;
2246     struct kvm_s390_float_interrupt *fi;
2247     struct kvm_s390_irq *buf;
2248     struct kvm_s390_irq *irq;
2249     int max_irqs;
2250     int ret = 0;
2251     int n = 0;
2252     int i;
2253
2254     if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
2255         return -EINVAL;
2256
2257     /*
2258      * We are already using -ENOMEM to signal
2259      * userspace it may retry with a bigger buffer,
2260      * so we need to use something else for this case
2261      */
2262     buf = vzalloc(len);
2263     if (!buf)
2264         return -ENOBUFS;
2265
2266     max_irqs = len / sizeof(struct kvm_s390_irq);
2267
2268     if (gi->origin && gisa_get_ipm(gi->origin)) {
2269         for (i = 0; i <= MAX_ISC; i++) {
2270             if (n == max_irqs) {
2271                 /* signal userspace to try again */
2272                 ret = -ENOMEM;
2273                 goto out_nolock;
2274             }
2275             if (gisa_tac_ipm_gisc(gi->origin, i)) {
2276                 irq = (struct kvm_s390_irq *) &buf[n];
2277                 irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
2278                 irq->u.io.io_int_word = isc_to_int_word(i);
2279                 n++;
2280             }
2281         }
2282     }
2283     fi = &kvm->arch.float_int;
2284     spin_lock(&fi->lock);
2285     for (i = 0; i < FIRQ_LIST_COUNT; i++) {
2286         list_for_each_entry(inti, &fi->lists[i], list) {
2287             if (n == max_irqs) {
2288                 /* signal userspace to try again */
2289                 ret = -ENOMEM;
2290                 goto out;
2291             }
2292             inti_to_irq(inti, &buf[n]);
2293             n++;
2294         }
2295     }
2296     if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) ||
2297         test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) {
2298         if (n == max_irqs) {
2299             /* signal userspace to try again */
2300             ret = -ENOMEM;
2301             goto out;
2302         }
2303         irq = (struct kvm_s390_irq *) &buf[n];
2304         irq->type = KVM_S390_INT_SERVICE;
2305         irq->u.ext = fi->srv_signal;
2306         n++;
2307     }
2308     if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
2309         if (n == max_irqs) {
2310                 /* signal userspace to try again */
2311                 ret = -ENOMEM;
2312                 goto out;
2313         }
2314         irq = (struct kvm_s390_irq *) &buf[n];
2315         irq->type = KVM_S390_MCHK;
2316         irq->u.mchk = fi->mchk;
2317         n++;
2318 }
2319
2320 out:
2321     spin_unlock(&fi->lock);
2322 out_nolock:
2323     if (!ret && n > 0) {
2324         if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
2325             ret = -EFAULT;
2326     }
2327     vfree(buf);
2328
2329     return ret < 0 ? ret : n;
2330 }
2331
2332 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
2333 {
2334     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2335     struct kvm_s390_ais_all ais;
2336
2337     if (attr->attr < sizeof(ais))
2338         return -EINVAL;
2339
2340     if (!test_kvm_facility(kvm, 72))
2341         return -EOPNOTSUPP;
2342
2343     mutex_lock(&fi->ais_lock);
2344     ais.simm = fi->simm;
2345     ais.nimm = fi->nimm;
2346     mutex_unlock(&fi->ais_lock);
2347
2348     if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais)))
2349         return -EFAULT;
2350
2351     return 0;
2352 }
2353
2354 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2355 {
2356     int r;
2357
2358     switch (attr->group) {
2359     case KVM_DEV_FLIC_GET_ALL_IRQS:
2360         r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
2361                       attr->attr);
2362         break;
2363     case KVM_DEV_FLIC_AISM_ALL:
2364         r = flic_ais_mode_get_all(dev->kvm, attr);
2365         break;
2366     default:
2367         r = -EINVAL;
2368     }
2369
2370     return r;
2371 }
2372
2373 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
2374                      u64 addr)
2375 {
2376     struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
2377     void *target = NULL;
2378     void __user *source;
2379     u64 size;
2380
2381     if (get_user(inti->type, (u64 __user *)addr))
2382         return -EFAULT;
2383
2384     switch (inti->type) {
2385     case KVM_S390_INT_PFAULT_INIT:
2386     case KVM_S390_INT_PFAULT_DONE:
2387     case KVM_S390_INT_VIRTIO:
2388     case KVM_S390_INT_SERVICE:
2389         target = (void *) &inti->ext;
2390         source = &uptr->u.ext;
2391         size = sizeof(inti->ext);
2392         break;
2393     case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
2394         target = (void *) &inti->io;
2395         source = &uptr->u.io;
2396         size = sizeof(inti->io);
2397         break;
2398     case KVM_S390_MCHK:
2399         target = (void *) &inti->mchk;
2400         source = &uptr->u.mchk;
2401         size = sizeof(inti->mchk);
2402         break;
2403     default:
2404         return -EINVAL;
2405     }
2406
2407     if (copy_from_user(target, source, size))
2408         return -EFAULT;
2409
2410     return 0;
2411 }
2412
2413 static int enqueue_floating_irq(struct kvm_device *dev,
2414                 struct kvm_device_attr *attr)
2415 {
2416     struct kvm_s390_interrupt_info *inti = NULL;
2417     int r = 0;
2418     int len = attr->attr;
2419
2420     if (len % sizeof(struct kvm_s390_irq) != 0)
2421         return -EINVAL;
2422     else if (len > KVM_S390_FLIC_MAX_BUFFER)
2423         return -EINVAL;
2424
2425     while (len >= sizeof(struct kvm_s390_irq)) {
2426         inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
2427         if (!inti)
2428             return -ENOMEM;
2429
2430         r = copy_irq_from_user(inti, attr->addr);
2431         if (r) {
2432             kfree(inti);
2433             return r;
2434         }
2435         r = __inject_vm(dev->kvm, inti);
2436         if (r) {
2437             kfree(inti);
2438             return r;
2439         }
2440         len -= sizeof(struct kvm_s390_irq);
2441         attr->addr += sizeof(struct kvm_s390_irq);
2442     }
2443
2444     return r;
2445 }
2446
2447 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
2448 {
2449     if (id >= MAX_S390_IO_ADAPTERS)
2450         return NULL;
2451     id = array_index_nospec(id, MAX_S390_IO_ADAPTERS);
2452     return kvm->arch.adapters[id];
2453 }
2454
2455 static int register_io_adapter(struct kvm_device *dev,
2456                    struct kvm_device_attr *attr)
2457 {
2458     struct s390_io_adapter *adapter;
2459     struct kvm_s390_io_adapter adapter_info;
2460
2461     if (copy_from_user(&adapter_info,
2462                (void __user *)attr->addr, sizeof(adapter_info)))
2463         return -EFAULT;
2464
2465     if (adapter_info.id >= MAX_S390_IO_ADAPTERS)
2466         return -EINVAL;
2467
2468     adapter_info.id = array_index_nospec(adapter_info.id,
2469                          MAX_S390_IO_ADAPTERS);
2470
2471     if (dev->kvm->arch.adapters[adapter_info.id] != NULL)
2472         return -EINVAL;
2473
2474     adapter = kzalloc(sizeof(*adapter), GFP_KERNEL_ACCOUNT);
2475     if (!adapter)
2476         return -ENOMEM;
2477
2478     adapter->id = adapter_info.id;
2479     adapter->isc = adapter_info.isc;
2480     adapter->maskable = adapter_info.maskable;
2481     adapter->masked = false;
2482     adapter->swap = adapter_info.swap;
2483     adapter->suppressible = (adapter_info.flags) &
2484                 KVM_S390_ADAPTER_SUPPRESSIBLE;
2485     dev->kvm->arch.adapters[adapter->id] = adapter;
2486
2487     return 0;
2488 }
2489
2490 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
2491 {
2492     int ret;
2493     struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2494
2495     if (!adapter || !adapter->maskable)
2496         return -EINVAL;
2497     ret = adapter->masked;
2498     adapter->masked = masked;
2499     return ret;
2500 }
2501
2502 void kvm_s390_destroy_adapters(struct kvm *kvm)
2503 {
2504     int i;
2505
2506     for (i = 0; i < MAX_S390_IO_ADAPTERS; i++)
2507         kfree(kvm->arch.adapters[i]);
2508 }
2509
2510 static int modify_io_adapter(struct kvm_device *dev,
2511                  struct kvm_device_attr *attr)
2512 {
2513     struct kvm_s390_io_adapter_req req;
2514     struct s390_io_adapter *adapter;
2515     int ret;
2516
2517     if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2518         return -EFAULT;
2519
2520     adapter = get_io_adapter(dev->kvm, req.id);
2521     if (!adapter)
2522         return -EINVAL;
2523     switch (req.type) {
2524     case KVM_S390_IO_ADAPTER_MASK:
2525         ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
2526         if (ret > 0)
2527             ret = 0;
2528         break;
2529     /*
2530      * The following operations are no longer needed and therefore no-ops.
2531      * The gpa to hva translation is done when an IRQ route is set up. The
2532      * set_irq code uses get_user_pages_remote() to do the actual write.
2533      */
2534     case KVM_S390_IO_ADAPTER_MAP:
2535     case KVM_S390_IO_ADAPTER_UNMAP:
2536         ret = 0;
2537         break;
2538     default:
2539         ret = -EINVAL;
2540     }
2541
2542     return ret;
2543 }
2544
2545 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
2546
2547 {
2548     const u64 isc_mask = 0xffUL << 24; /* all iscs set */
2549     u32 schid;
2550
2551     if (attr->flags)
2552         return -EINVAL;
2553     if (attr->attr != sizeof(schid))
2554         return -EINVAL;
2555     if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid)))
2556         return -EFAULT;
2557     if (!schid)
2558         return -EINVAL;
2559     kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
2560     /*
2561      * If userspace is conforming to the architecture, we can have at most
2562      * one pending I/O interrupt per subchannel, so this is effectively a
2563      * clear all.
2564      */
2565     return 0;
2566 }
2567
2568 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
2569 {
2570     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2571     struct kvm_s390_ais_req req;
2572     int ret = 0;
2573
2574     if (!test_kvm_facility(kvm, 72))
2575         return -EOPNOTSUPP;
2576
2577     if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
2578         return -EFAULT;
2579
2580     if (req.isc > MAX_ISC)
2581         return -EINVAL;
2582
2583     trace_kvm_s390_modify_ais_mode(req.isc,
2584                        (fi->simm & AIS_MODE_MASK(req.isc)) ?
2585                        (fi->nimm & AIS_MODE_MASK(req.isc)) ?
2586                        2 : KVM_S390_AIS_MODE_SINGLE :
2587                        KVM_S390_AIS_MODE_ALL, req.mode);
2588
2589     mutex_lock(&fi->ais_lock);
2590     switch (req.mode) {
2591     case KVM_S390_AIS_MODE_ALL:
2592         fi->simm &= ~AIS_MODE_MASK(req.isc);
2593         fi->nimm &= ~AIS_MODE_MASK(req.isc);
2594         break;
2595     case KVM_S390_AIS_MODE_SINGLE:
2596         fi->simm |= AIS_MODE_MASK(req.isc);
2597         fi->nimm &= ~AIS_MODE_MASK(req.isc);
2598         break;
2599     default:
2600         ret = -EINVAL;
2601     }
2602     mutex_unlock(&fi->ais_lock);
2603
2604     return ret;
2605 }
2606
2607 static int kvm_s390_inject_airq(struct kvm *kvm,
2608                 struct s390_io_adapter *adapter)
2609 {
2610     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2611     struct kvm_s390_interrupt s390int = {
2612         .type = KVM_S390_INT_IO(1, 0, 0, 0),
2613         .parm = 0,
2614         .parm64 = isc_to_int_word(adapter->isc),
2615     };
2616     int ret = 0;
2617
2618     if (!test_kvm_facility(kvm, 72) || !adapter->suppressible)
2619         return kvm_s390_inject_vm(kvm, &s390int);
2620
2621     mutex_lock(&fi->ais_lock);
2622     if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
2623         trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc);
2624         goto out;
2625     }
2626
2627     ret = kvm_s390_inject_vm(kvm, &s390int);
2628     if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
2629         fi->nimm |= AIS_MODE_MASK(adapter->isc);
2630         trace_kvm_s390_modify_ais_mode(adapter->isc,
2631                            KVM_S390_AIS_MODE_SINGLE, 2);
2632     }
2633 out:
2634     mutex_unlock(&fi->ais_lock);
2635     return ret;
2636 }
2637
2638 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
2639 {
2640     unsigned int id = attr->attr;
2641     struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
2642
2643     if (!adapter)
2644         return -EINVAL;
2645
2646     return kvm_s390_inject_airq(kvm, adapter);
2647 }
2648
2649 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
2650 {
2651     struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
2652     struct kvm_s390_ais_all ais;
2653
2654     if (!test_kvm_facility(kvm, 72))
2655         return -EOPNOTSUPP;
2656
2657     if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais)))
2658         return -EFAULT;
2659
2660     mutex_lock(&fi->ais_lock);
2661     fi->simm = ais.simm;
2662     fi->nimm = ais.nimm;
2663     mutex_unlock(&fi->ais_lock);
2664
2665     return 0;
2666 }
2667
2668 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
2669 {
2670     int r = 0;
2671     unsigned long i;
2672     struct kvm_vcpu *vcpu;
2673
2674     switch (attr->group) {
2675     case KVM_DEV_FLIC_ENQUEUE:
2676         r = enqueue_floating_irq(dev, attr);
2677         break;
2678     case KVM_DEV_FLIC_CLEAR_IRQS:
2679         kvm_s390_clear_float_irqs(dev->kvm);
2680         break;
2681     case KVM_DEV_FLIC_APF_ENABLE:
2682         dev->kvm->arch.gmap->pfault_enabled = 1;
2683         break;
2684     case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2685         dev->kvm->arch.gmap->pfault_enabled = 0;
2686         /*
2687          * Make sure no async faults are in transition when
2688          * clearing the queues. So we don't need to worry
2689          * about late coming workers.
2690          */
2691         synchronize_srcu(&dev->kvm->srcu);
2692         kvm_for_each_vcpu(i, vcpu, dev->kvm)
2693             kvm_clear_async_pf_completion_queue(vcpu);
2694         break;
2695     case KVM_DEV_FLIC_ADAPTER_REGISTER:
2696         r = register_io_adapter(dev, attr);
2697         break;
2698     case KVM_DEV_FLIC_ADAPTER_MODIFY:
2699         r = modify_io_adapter(dev, attr);
2700         break;
2701     case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2702         r = clear_io_irq(dev->kvm, attr);
2703         break;
2704     case KVM_DEV_FLIC_AISM:
2705         r = modify_ais_mode(dev->kvm, attr);
2706         break;
2707     case KVM_DEV_FLIC_AIRQ_INJECT:
2708         r = flic_inject_airq(dev->kvm, attr);
2709         break;
2710     case KVM_DEV_FLIC_AISM_ALL:
2711         r = flic_ais_mode_set_all(dev->kvm, attr);
2712         break;
2713     default:
2714         r = -EINVAL;
2715     }
2716
2717     return r;
2718 }
2719
2720 static int flic_has_attr(struct kvm_device *dev,
2721                  struct kvm_device_attr *attr)
2722 {
2723     switch (attr->group) {
2724     case KVM_DEV_FLIC_GET_ALL_IRQS:
2725     case KVM_DEV_FLIC_ENQUEUE:
2726     case KVM_DEV_FLIC_CLEAR_IRQS:
2727     case KVM_DEV_FLIC_APF_ENABLE:
2728     case KVM_DEV_FLIC_APF_DISABLE_WAIT:
2729     case KVM_DEV_FLIC_ADAPTER_REGISTER:
2730     case KVM_DEV_FLIC_ADAPTER_MODIFY:
2731     case KVM_DEV_FLIC_CLEAR_IO_IRQ:
2732     case KVM_DEV_FLIC_AISM:
2733     case KVM_DEV_FLIC_AIRQ_INJECT:
2734     case KVM_DEV_FLIC_AISM_ALL:
2735         return 0;
2736     }
2737     return -ENXIO;
2738 }
2739
2740 static int flic_create(struct kvm_device *dev, u32 type)
2741 {
2742     if (!dev)
2743         return -EINVAL;
2744     if (dev->kvm->arch.flic)
2745         return -EINVAL;
2746     dev->kvm->arch.flic = dev;
2747     return 0;
2748 }
2749
2750 static void flic_destroy(struct kvm_device *dev)
2751 {
2752     dev->kvm->arch.flic = NULL;
2753     kfree(dev);
2754 }
2755
2756 /* s390 floating irq controller (flic) */
2757 struct kvm_device_ops kvm_flic_ops = {
2758     .name = "kvm-flic",
2759     .get_attr = flic_get_attr,
2760     .set_attr = flic_set_attr,
2761     .has_attr = flic_has_attr,
2762     .create = flic_create,
2763     .destroy = flic_destroy,
2764 };
2765
2766 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
2767 {
2768     unsigned long bit;
2769
2770     bit = bit_nr + (addr % PAGE_SIZE) * 8;
2771
2772     return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
2773 }
2774
2775 static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
2776 {
2777     struct page *page = NULL;
2778
2779     mmap_read_lock(kvm->mm);
2780     get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE,
2781                   &page, NULL, NULL);
2782     mmap_read_unlock(kvm->mm);
2783     return page;
2784 }
2785
2786 static int adapter_indicators_set(struct kvm *kvm,
2787                   struct s390_io_adapter *adapter,
2788                   struct kvm_s390_adapter_int *adapter_int)
2789 {
2790     unsigned long bit;
2791     int summary_set, idx;
2792     struct page *ind_page, *summary_page;
2793     void *map;
2794
2795     ind_page = get_map_page(kvm, adapter_int->ind_addr);
2796     if (!ind_page)
2797         return -1;
2798     summary_page = get_map_page(kvm, adapter_int->summary_addr);
2799     if (!summary_page) {
2800         put_page(ind_page);
2801         return -1;
2802     }
2803
2804     idx = srcu_read_lock(&kvm->srcu);
2805     map = page_address(ind_page);
2806     bit = get_ind_bit(adapter_int->ind_addr,
2807               adapter_int->ind_offset, adapter->swap);
2808     set_bit(bit, map);
2809     mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT);
2810     set_page_dirty_lock(ind_page);
2811     map = page_address(summary_page);
2812     bit = get_ind_bit(adapter_int->summary_addr,
2813               adapter_int->summary_offset, adapter->swap);
2814     summary_set = test_and_set_bit(bit, map);
2815     mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT);
2816     set_page_dirty_lock(summary_page);
2817     srcu_read_unlock(&kvm->srcu, idx);
2818
2819     put_page(ind_page);
2820     put_page(summary_page);
2821     return summary_set ? 0 : 1;
2822 }
2823
2824 /*
2825  * < 0 - not injected due to error
2826  * = 0 - coalesced, summary indicator already active
2827  * > 0 - injected interrupt
2828  */
2829 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
2830                struct kvm *kvm, int irq_source_id, int level,
2831                bool line_status)
2832 {
2833     int ret;
2834     struct s390_io_adapter *adapter;
2835
2836     /* We're only interested in the 0->1 transition. */
2837     if (!level)
2838         return 0;
2839     adapter = get_io_adapter(kvm, e->adapter.adapter_id);
2840     if (!adapter)
2841         return -1;
2842     ret = adapter_indicators_set(kvm, adapter, &e->adapter);
2843     if ((ret > 0) && !adapter->masked) {
2844         ret = kvm_s390_inject_airq(kvm, adapter);
2845         if (ret == 0)
2846             ret = 1;
2847     }
2848     return ret;
2849 }
2850
2851 /*
2852  * Inject the machine check to the guest.
2853  */
2854 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
2855                      struct mcck_volatile_info *mcck_info)
2856 {
2857     struct kvm_s390_interrupt_info inti;
2858     struct kvm_s390_irq irq;
2859     struct kvm_s390_mchk_info *mchk;
2860     union mci mci;
2861     __u64 cr14 = 0;         /* upper bits are not used */
2862     int rc;
2863
2864     mci.val = mcck_info->mcic;
2865     if (mci.sr)
2866         cr14 |= CR14_RECOVERY_SUBMASK;
2867     if (mci.dg)
2868         cr14 |= CR14_DEGRADATION_SUBMASK;
2869     if (mci.w)
2870         cr14 |= CR14_WARNING_SUBMASK;
2871
2872     mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
2873     mchk->cr14 = cr14;
2874     mchk->mcic = mcck_info->mcic;
2875     mchk->ext_damage_code = mcck_info->ext_damage_code;
2876     mchk->failing_storage_address = mcck_info->failing_storage_address;
2877     if (mci.ck) {
2878         /* Inject the floating machine check */
2879         inti.type = KVM_S390_MCHK;
2880         rc = __inject_vm(vcpu->kvm, &inti);
2881     } else {
2882         /* Inject the machine check to specified vcpu */
2883         irq.type = KVM_S390_MCHK;
2884         rc = kvm_s390_inject_vcpu(vcpu, &irq);
2885     }
2886     WARN_ON_ONCE(rc);
2887 }
2888
2889 int kvm_set_routing_entry(struct kvm *kvm,
2890               struct kvm_kernel_irq_routing_entry *e,
2891               const struct kvm_irq_routing_entry *ue)
2892 {
2893     u64 uaddr;
2894
2895     switch (ue->type) {
2896     /* we store the userspace addresses instead of the guest addresses */
2897     case KVM_IRQ_ROUTING_S390_ADAPTER:
2898         e->set = set_adapter_int;
2899         uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr);
2900         if (uaddr == -EFAULT)
2901             return -EFAULT;
2902         e->adapter.summary_addr = uaddr;
2903         uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr);
2904         if (uaddr == -EFAULT)
2905             return -EFAULT;
2906         e->adapter.ind_addr = uaddr;
2907         e->adapter.summary_offset = ue->u.adapter.summary_offset;
2908         e->adapter.ind_offset = ue->u.adapter.ind_offset;
2909         e->adapter.adapter_id = ue->u.adapter.adapter_id;
2910         return 0;
2911     default:
2912         return -EINVAL;
2913     }
2914 }
2915
2916 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
2917         int irq_source_id, int level, bool line_status)
2918 {
2919     return -EINVAL;
2920 }
2921
2922 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
2923 {
2924     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
2925     struct kvm_s390_irq *buf;
2926     int r = 0;
2927     int n;
2928
2929     buf = vmalloc(len);
2930     if (!buf)
2931         return -ENOMEM;
2932
2933     if (copy_from_user((void *) buf, irqstate, len)) {
2934         r = -EFAULT;
2935         goto out_free;
2936     }
2937
2938     /*
2939      * Don't allow setting the interrupt state
2940      * when there are already interrupts pending
2941      */
2942     spin_lock(&li->lock);
2943     if (li->pending_irqs) {
2944         r = -EBUSY;
2945         goto out_unlock;
2946     }
2947
2948     for (n = 0; n < len / sizeof(*buf); n++) {
2949         r = do_inject_vcpu(vcpu, &buf[n]);
2950         if (r)
2951             break;
2952     }
2953
2954 out_unlock:
2955     spin_unlock(&li->lock);
2956 out_free:
2957     vfree(buf);
2958
2959     return r;
2960 }
2961
2962 static void store_local_irq(struct kvm_s390_local_interrupt *li,
2963                 struct kvm_s390_irq *irq,
2964                 unsigned long irq_type)
2965 {
2966     switch (irq_type) {
2967     case IRQ_PEND_MCHK_EX:
2968     case IRQ_PEND_MCHK_REP:
2969         irq->type = KVM_S390_MCHK;
2970         irq->u.mchk = li->irq.mchk;
2971         break;
2972     case IRQ_PEND_PROG:
2973         irq->type = KVM_S390_PROGRAM_INT;
2974         irq->u.pgm = li->irq.pgm;
2975         break;
2976     case IRQ_PEND_PFAULT_INIT:
2977         irq->type = KVM_S390_INT_PFAULT_INIT;
2978         irq->u.ext = li->irq.ext;
2979         break;
2980     case IRQ_PEND_EXT_EXTERNAL:
2981         irq->type = KVM_S390_INT_EXTERNAL_CALL;
2982         irq->u.extcall = li->irq.extcall;
2983         break;
2984     case IRQ_PEND_EXT_CLOCK_COMP:
2985         irq->type = KVM_S390_INT_CLOCK_COMP;
2986         break;
2987     case IRQ_PEND_EXT_CPU_TIMER:
2988         irq->type = KVM_S390_INT_CPU_TIMER;
2989         break;
2990     case IRQ_PEND_SIGP_STOP:
2991         irq->type = KVM_S390_SIGP_STOP;
2992         irq->u.stop = li->irq.stop;
2993         break;
2994     case IRQ_PEND_RESTART:
2995         irq->type = KVM_S390_RESTART;
2996         break;
2997     case IRQ_PEND_SET_PREFIX:
2998         irq->type = KVM_S390_SIGP_SET_PREFIX;
2999         irq->u.prefix = li->irq.prefix;
3000         break;
3001     }
3002 }
3003
3004 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
3005 {
3006     int scn;
3007     DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
3008     struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
3009     unsigned long pending_irqs;
3010     struct kvm_s390_irq irq;
3011     unsigned long irq_type;
3012     int cpuaddr;
3013     int n = 0;
3014
3015     spin_lock(&li->lock);
3016     pending_irqs = li->pending_irqs;
3017     memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
3018            sizeof(sigp_emerg_pending));
3019     spin_unlock(&li->lock);
3020
3021     for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
3022         memset(&irq, 0, sizeof(irq));
3023         if (irq_type == IRQ_PEND_EXT_EMERGENCY)
3024             continue;
3025         if (n + sizeof(irq) > len)
3026             return -ENOBUFS;
3027         store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
3028         if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3029             return -EFAULT;
3030         n += sizeof(irq);
3031     }
3032
3033     if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
3034         for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
3035             memset(&irq, 0, sizeof(irq));
3036             if (n + sizeof(irq) > len)
3037                 return -ENOBUFS;
3038             irq.type = KVM_S390_INT_EMERGENCY;
3039             irq.u.emerg.code = cpuaddr;
3040             if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3041                 return -EFAULT;
3042             n += sizeof(irq);
3043         }
3044     }
3045
3046     if (sca_ext_call_pending(vcpu, &scn)) {
3047         if (n + sizeof(irq) > len)
3048             return -ENOBUFS;
3049         memset(&irq, 0, sizeof(irq));
3050         irq.type = KVM_S390_INT_EXTERNAL_CALL;
3051         irq.u.extcall.code = scn;
3052         if (copy_to_user(&buf[n], &irq, sizeof(irq)))
3053             return -EFAULT;
3054         n += sizeof(irq);
3055     }
3056
3057     return n;
3058 }
3059
3060 static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
3061 {
3062     int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus);
3063     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3064     struct kvm_vcpu *vcpu;
3065     u8 vcpu_isc_mask;
3066
3067     for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
3068         vcpu = kvm_get_vcpu(kvm, vcpu_idx);
3069         if (psw_ioint_disabled(vcpu))
3070             continue;
3071         vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24);
3072         if (deliverable_mask & vcpu_isc_mask) {
3073             /* lately kicked but not yet running */
3074             if (test_and_set_bit(vcpu_idx, gi->kicked_mask))
3075                 return;
3076             kvm_s390_vcpu_wakeup(vcpu);
3077             return;
3078         }
3079     }
3080 }
3081
3082 static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer)
3083 {
3084     struct kvm_s390_gisa_interrupt *gi =
3085         container_of(timer, struct kvm_s390_gisa_interrupt, timer);
3086     struct kvm *kvm =
3087         container_of(gi->origin, struct sie_page2, gisa)->kvm;
3088     u8 pending_mask;
3089
3090     pending_mask = gisa_get_ipm_or_restore_iam(gi);
3091     if (pending_mask) {
3092         __airqs_kick_single_vcpu(kvm, pending_mask);
3093         hrtimer_forward_now(timer, ns_to_ktime(gi->expires));
3094         return HRTIMER_RESTART;
3095     }
3096
3097     return HRTIMER_NORESTART;
3098 }
3099
3100 #define NULL_GISA_ADDR 0x00000000UL
3101 #define NONE_GISA_ADDR 0x00000001UL
3102 #define GISA_ADDR_MASK 0xfffff000UL
3103
3104 static void process_gib_alert_list(void)
3105 {
3106     struct kvm_s390_gisa_interrupt *gi;
3107     struct kvm_s390_gisa *gisa;
3108     struct kvm *kvm;
3109     u32 final, origin = 0UL;
3110
3111     do {
3112         /*
3113          * If the NONE_GISA_ADDR is still stored in the alert list
3114          * origin, we will leave the outer loop. No further GISA has
3115          * been added to the alert list by millicode while processing
3116          * the current alert list.
3117          */
3118         final = (origin & NONE_GISA_ADDR);
3119         /*
3120          * Cut off the alert list and store the NONE_GISA_ADDR in the
3121          * alert list origin to avoid further GAL interruptions.
3122          * A new alert list can be build up by millicode in parallel
3123          * for guests not in the yet cut-off alert list. When in the
3124          * final loop, store the NULL_GISA_ADDR instead. This will re-
3125          * enable GAL interruptions on the host again.
3126          */
3127         origin = xchg(&gib->alert_list_origin,
3128                   (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR);
3129         /*
3130          * Loop through the just cut-off alert list and start the
3131          * gisa timers to kick idle vcpus to consume the pending
3132          * interruptions asap.
3133          */
3134         while (origin & GISA_ADDR_MASK) {
3135             gisa = (struct kvm_s390_gisa *)(u64)origin;
3136             origin = gisa->next_alert;
3137             gisa->next_alert = (u32)(u64)gisa;
3138             kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
3139             gi = &kvm->arch.gisa_int;
3140             if (hrtimer_active(&gi->timer))
3141                 hrtimer_cancel(&gi->timer);
3142             hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3143         }
3144     } while (!final);
3145
3146 }
3147
3148 void kvm_s390_gisa_clear(struct kvm *kvm)
3149 {
3150     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3151
3152     if (!gi->origin)
3153         return;
3154     gisa_clear_ipm(gi->origin);
3155     VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin);
3156 }
3157
3158 void kvm_s390_gisa_init(struct kvm *kvm)
3159 {
3160     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3161
3162     if (!css_general_characteristics.aiv)
3163         return;
3164     gi->origin = &kvm->arch.sie_page2->gisa;
3165     gi->alert.mask = 0;
3166     spin_lock_init(&gi->alert.ref_lock);
3167     gi->expires = 50 * 1000; /* 50 usec */
3168     hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3169     gi->timer.function = gisa_vcpu_kicker;
3170     memset(gi->origin, 0, sizeof(struct kvm_s390_gisa));
3171     gi->origin->next_alert = (u32)(u64)gi->origin;
3172     VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin);
3173 }
3174
3175 void kvm_s390_gisa_enable(struct kvm *kvm)
3176 {
3177     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3178     struct kvm_vcpu *vcpu;
3179     unsigned long i;
3180     u32 gisa_desc;
3181
3182     if (gi->origin)
3183         return;
3184     kvm_s390_gisa_init(kvm);
3185     gisa_desc = kvm_s390_get_gisa_desc(kvm);
3186     if (!gisa_desc)
3187         return;
3188     kvm_for_each_vcpu(i, vcpu, kvm) {
3189         mutex_lock(&vcpu->mutex);
3190         vcpu->arch.sie_block->gd = gisa_desc;
3191         vcpu->arch.sie_block->eca |= ECA_AIV;
3192         VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3193                vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3194         mutex_unlock(&vcpu->mutex);
3195     }
3196 }
3197
3198 void kvm_s390_gisa_destroy(struct kvm *kvm)
3199 {
3200     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3201     struct kvm_s390_gisa *gisa = gi->origin;
3202
3203     if (!gi->origin)
3204         return;
3205     if (gi->alert.mask)
3206         KVM_EVENT(3, "vm 0x%pK has unexpected iam 0x%02x",
3207               kvm, gi->alert.mask);
3208     while (gisa_in_alert_list(gi->origin))
3209         cpu_relax();
3210     hrtimer_cancel(&gi->timer);
3211     gi->origin = NULL;
3212     VM_EVENT(kvm, 3, "gisa 0x%pK destroyed", gisa);
3213 }
3214
3215 void kvm_s390_gisa_disable(struct kvm *kvm)
3216 {
3217     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3218     struct kvm_vcpu *vcpu;
3219     unsigned long i;
3220
3221     if (!gi->origin)
3222         return;
3223     kvm_for_each_vcpu(i, vcpu, kvm) {
3224         mutex_lock(&vcpu->mutex);
3225         vcpu->arch.sie_block->eca &= ~ECA_AIV;
3226         vcpu->arch.sie_block->gd = 0U;
3227         mutex_unlock(&vcpu->mutex);
3228         VCPU_EVENT(vcpu, 3, "AIV disabled for cpu %03u", vcpu->vcpu_id);
3229     }
3230     kvm_s390_gisa_destroy(kvm);
3231 }
3232
3233 /**
3234  * kvm_s390_gisc_register - register a guest ISC
3235  *
3236  * @kvm:  the kernel vm to work with
3237  * @gisc: the guest interruption sub class to register
3238  *
3239  * The function extends the vm specific alert mask to use.
3240  * The effective IAM mask in the GISA is updated as well
3241  * in case the GISA is not part of the GIB alert list.
3242  * It will be updated latest when the IAM gets restored
3243  * by gisa_get_ipm_or_restore_iam().
3244  *
3245  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3246  *          has registered with the channel subsystem.
3247  *          -ENODEV in case the vm uses no GISA
3248  *          -ERANGE in case the guest ISC is invalid
3249  */
3250 int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
3251 {
3252     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3253
3254     if (!gi->origin)
3255         return -ENODEV;
3256     if (gisc > MAX_ISC)
3257         return -ERANGE;
3258
3259     spin_lock(&gi->alert.ref_lock);
3260     gi->alert.ref_count[gisc]++;
3261     if (gi->alert.ref_count[gisc] == 1) {
3262         gi->alert.mask |= 0x80 >> gisc;
3263         gisa_set_iam(gi->origin, gi->alert.mask);
3264     }
3265     spin_unlock(&gi->alert.ref_lock);
3266
3267     return gib->nisc;
3268 }
3269 EXPORT_SYMBOL_GPL(kvm_s390_gisc_register);
3270
3271 /**
3272  * kvm_s390_gisc_unregister - unregister a guest ISC
3273  *
3274  * @kvm:  the kernel vm to work with
3275  * @gisc: the guest interruption sub class to register
3276  *
3277  * The function reduces the vm specific alert mask to use.
3278  * The effective IAM mask in the GISA is updated as well
3279  * in case the GISA is not part of the GIB alert list.
3280  * It will be updated latest when the IAM gets restored
3281  * by gisa_get_ipm_or_restore_iam().
3282  *
3283  * Returns: the nonspecific ISC (NISC) the gib alert mechanism
3284  *          has registered with the channel subsystem.
3285  *          -ENODEV in case the vm uses no GISA
3286  *          -ERANGE in case the guest ISC is invalid
3287  *          -EINVAL in case the guest ISC is not registered
3288  */
3289 int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
3290 {
3291     struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
3292     int rc = 0;
3293
3294     if (!gi->origin)
3295         return -ENODEV;
3296     if (gisc > MAX_ISC)
3297         return -ERANGE;
3298
3299     spin_lock(&gi->alert.ref_lock);
3300     if (gi->alert.ref_count[gisc] == 0) {
3301         rc = -EINVAL;
3302         goto out;
3303     }
3304     gi->alert.ref_count[gisc]--;
3305     if (gi->alert.ref_count[gisc] == 0) {
3306         gi->alert.mask &= ~(0x80 >> gisc);
3307         gisa_set_iam(gi->origin, gi->alert.mask);
3308     }
3309 out:
3310     spin_unlock(&gi->alert.ref_lock);
3311
3312     return rc;
3313 }
3314 EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister);
3315
3316 static void aen_host_forward(unsigned long si)
3317 {
3318     struct kvm_s390_gisa_interrupt *gi;
3319     struct zpci_gaite *gaite;
3320     struct kvm *kvm;
3321
3322     gaite = (struct zpci_gaite *)aift->gait +
3323         (si * sizeof(struct zpci_gaite));
3324     if (gaite->count == 0)
3325         return;
3326     if (gaite->aisb != 0)
3327         set_bit_inv(gaite->aisbo, phys_to_virt(gaite->aisb));
3328
3329     kvm = kvm_s390_pci_si_to_kvm(aift, si);
3330     if (!kvm)
3331         return;
3332     gi = &kvm->arch.gisa_int;
3333
3334     if (!(gi->origin->g1.simm & AIS_MODE_MASK(gaite->gisc)) ||
3335         !(gi->origin->g1.nimm & AIS_MODE_MASK(gaite->gisc))) {
3336         gisa_set_ipm_gisc(gi->origin, gaite->gisc);
3337         if (hrtimer_active(&gi->timer))
3338             hrtimer_cancel(&gi->timer);
3339         hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
3340         kvm->stat.aen_forward++;
3341     }
3342 }
3343
3344 static void aen_process_gait(u8 isc)
3345 {
3346     bool found = false, first = true;
3347     union zpci_sic_iib iib = {{0}};
3348     unsigned long si, flags;
3349
3350     spin_lock_irqsave(&aift->gait_lock, flags);
3351
3352     if (!aift->gait) {
3353         spin_unlock_irqrestore(&aift->gait_lock, flags);
3354         return;
3355     }
3356
3357     for (si = 0;;) {
3358         /* Scan adapter summary indicator bit vector */
3359         si = airq_iv_scan(aift->sbv, si, airq_iv_end(aift->sbv));
3360         if (si == -1UL) {
3361             if (first || found) {
3362                 /* Re-enable interrupts. */
3363                 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, isc,
3364                           &iib);
3365                 first = found = false;
3366             } else {
3367                 /* Interrupts on and all bits processed */
3368                 break;
3369             }
3370             found = false;
3371             si = 0;
3372             /* Scan again after re-enabling interrupts */
3373             continue;
3374         }
3375         found = true;
3376         aen_host_forward(si);
3377     }
3378
3379     spin_unlock_irqrestore(&aift->gait_lock, flags);
3380 }
3381
3382 static void gib_alert_irq_handler(struct airq_struct *airq,
3383                   struct tpi_info *tpi_info)
3384 {
3385     struct tpi_adapter_info *info = (struct tpi_adapter_info *)tpi_info;
3386
3387     inc_irq_stat(IRQIO_GAL);
3388
3389     if ((info->forward || info->error) &&
3390         IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3391         aen_process_gait(info->isc);
3392         if (info->aism != 0)
3393             process_gib_alert_list();
3394     } else {
3395         process_gib_alert_list();
3396     }
3397 }
3398
3399 static struct airq_struct gib_alert_irq = {
3400     .handler = gib_alert_irq_handler,
3401     .lsi_ptr = &gib_alert_irq.lsi_mask,
3402 };
3403
3404 void kvm_s390_gib_destroy(void)
3405 {
3406     if (!gib)
3407         return;
3408     if (kvm_s390_pci_interp_allowed() && aift) {
3409         mutex_lock(&aift->aift_lock);
3410         kvm_s390_pci_aen_exit();
3411         mutex_unlock(&aift->aift_lock);
3412     }
3413     chsc_sgib(0);
3414     unregister_adapter_interrupt(&gib_alert_irq);
3415     free_page((unsigned long)gib);
3416     gib = NULL;
3417 }
3418
3419 int kvm_s390_gib_init(u8 nisc)
3420 {
3421     int rc = 0;
3422
3423     if (!css_general_characteristics.aiv) {
3424         KVM_EVENT(3, "%s", "gib not initialized, no AIV facility");
3425         goto out;
3426     }
3427
3428     gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3429     if (!gib) {
3430         rc = -ENOMEM;
3431         goto out;
3432     }
3433
3434     gib_alert_irq.isc = nisc;
3435     if (register_adapter_interrupt(&gib_alert_irq)) {
3436         pr_err("Registering the GIB alert interruption handler failed\n");
3437         rc = -EIO;
3438         goto out_free_gib;
3439     }
3440
3441     gib->nisc = nisc;
3442     if (chsc_sgib((u32)(u64)gib)) {
3443         pr_err("Associating the GIB with the AIV facility failed\n");
3444         free_page((unsigned long)gib);
3445         gib = NULL;
3446         rc = -EIO;
3447         goto out_unreg_gal;
3448     }
3449
3450     if (kvm_s390_pci_interp_allowed()) {
3451         if (kvm_s390_pci_aen_init(nisc)) {
3452             pr_err("Initializing AEN for PCI failed\n");
3453             rc = -EIO;
3454             goto out_unreg_gal;
3455         }
3456     }
3457
3458     KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc);
3459     goto out;
3460
3461 out_unreg_gal:
3462     unregister_adapter_interrupt(&gib_alert_irq);
3463 out_free_gib:
3464     free_page((unsigned long)gib);
3465     gib = NULL;
3466 out:
3467     return rc;
3468 }