x86/kvm/irq_comm.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * irq_comm.c: Common API for in kernel interrupt controller
0004  * Copyright (c) 2007, Intel Corporation.
0005  *
0006  * Authors:
0007  *   Yaozu (Eddie) Dong <Eddie.dong@intel.com>
0008  *
0009  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
0010  */
0011
0012 #include <linux/kvm_host.h>
0013 #include <linux/slab.h>
0014 #include <linux/export.h>
0015 #include <linux/rculist.h>
0016
0017 #include <trace/events/kvm.h>
0018
0019 #include "irq.h"
0020
0021 #include "ioapic.h"
0022
0023 #include "lapic.h"
0024
0025 #include "hyperv.h"
0026 #include "x86.h"
0027 #include "xen.h"
0028
0029 static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e,
0030                struct kvm *kvm, int irq_source_id, int level,
0031                bool line_status)
0032 {
0033     struct kvm_pic *pic = kvm->arch.vpic;
0034     return kvm_pic_set_irq(pic, e->irqchip.pin, irq_source_id, level);
0035 }
0036
0037 static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e,
0038                   struct kvm *kvm, int irq_source_id, int level,
0039                   bool line_status)
0040 {
0041     struct kvm_ioapic *ioapic = kvm->arch.vioapic;
0042     return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, irq_source_id, level,
0043                 line_status);
0044 }
0045
0046 int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
0047         struct kvm_lapic_irq *irq, struct dest_map *dest_map)
0048 {
0049     int r = -1;
0050     struct kvm_vcpu *vcpu, *lowest = NULL;
0051     unsigned long i, dest_vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)];
0052     unsigned int dest_vcpus = 0;
0053
0054     if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, dest_map))
0055         return r;
0056
0057     if (irq->dest_mode == APIC_DEST_PHYSICAL &&
0058         irq->dest_id == 0xff && kvm_lowest_prio_delivery(irq)) {
0059         printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio\n");
0060         irq->delivery_mode = APIC_DM_FIXED;
0061     }
0062
0063     memset(dest_vcpu_bitmap, 0, sizeof(dest_vcpu_bitmap));
0064
0065     kvm_for_each_vcpu(i, vcpu, kvm) {
0066         if (!kvm_apic_present(vcpu))
0067             continue;
0068
0069         if (!kvm_apic_match_dest(vcpu, src, irq->shorthand,
0070                     irq->dest_id, irq->dest_mode))
0071             continue;
0072
0073         if (!kvm_lowest_prio_delivery(irq)) {
0074             if (r < 0)
0075                 r = 0;
0076             r += kvm_apic_set_irq(vcpu, irq, dest_map);
0077         } else if (kvm_apic_sw_enabled(vcpu->arch.apic)) {
0078             if (!kvm_vector_hashing_enabled()) {
0079                 if (!lowest)
0080                     lowest = vcpu;
0081                 else if (kvm_apic_compare_prio(vcpu, lowest) < 0)
0082                     lowest = vcpu;
0083             } else {
0084                 __set_bit(i, dest_vcpu_bitmap);
0085                 dest_vcpus++;
0086             }
0087         }
0088     }
0089
0090     if (dest_vcpus != 0) {
0091         int idx = kvm_vector_to_index(irq->vector, dest_vcpus,
0092                     dest_vcpu_bitmap, KVM_MAX_VCPUS);
0093
0094         lowest = kvm_get_vcpu(kvm, idx);
0095     }
0096
0097     if (lowest)
0098         r = kvm_apic_set_irq(lowest, irq, dest_map);
0099
0100     return r;
0101 }
0102
0103 void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
0104              struct kvm_lapic_irq *irq)
0105 {
0106     struct msi_msg msg = { .address_lo = e->msi.address_lo,
0107                    .address_hi = e->msi.address_hi,
0108                    .data = e->msi.data };
0109
0110     trace_kvm_msi_set_irq(msg.address_lo | (kvm->arch.x2apic_format ?
0111                   (u64)msg.address_hi << 32 : 0), msg.data);
0112
0113     irq->dest_id = x86_msi_msg_get_destid(&msg, kvm->arch.x2apic_format);
0114     irq->vector = msg.arch_data.vector;
0115     irq->dest_mode = kvm_lapic_irq_dest_mode(msg.arch_addr_lo.dest_mode_logical);
0116     irq->trig_mode = msg.arch_data.is_level;
0117     irq->delivery_mode = msg.arch_data.delivery_mode << 8;
0118     irq->msi_redir_hint = msg.arch_addr_lo.redirect_hint;
0119     irq->level = 1;
0120     irq->shorthand = APIC_DEST_NOSHORT;
0121 }
0122 EXPORT_SYMBOL_GPL(kvm_set_msi_irq);
0123
0124 static inline bool kvm_msi_route_invalid(struct kvm *kvm,
0125         struct kvm_kernel_irq_routing_entry *e)
0126 {
0127     return kvm->arch.x2apic_format && (e->msi.address_hi & 0xff);
0128 }
0129
0130 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e,
0131         struct kvm *kvm, int irq_source_id, int level, bool line_status)
0132 {
0133     struct kvm_lapic_irq irq;
0134
0135     if (kvm_msi_route_invalid(kvm, e))
0136         return -EINVAL;
0137
0138     if (!level)
0139         return -1;
0140
0141     kvm_set_msi_irq(kvm, e, &irq);
0142
0143     return kvm_irq_delivery_to_apic(kvm, NULL, &irq, NULL);
0144 }
0145
0146
0147 static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry *e,
0148             struct kvm *kvm, int irq_source_id, int level,
0149             bool line_status)
0150 {
0151     if (!level)
0152         return -1;
0153
0154     return kvm_hv_synic_set_irq(kvm, e->hv_sint.vcpu, e->hv_sint.sint);
0155 }
0156
0157 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e,
0158                   struct kvm *kvm, int irq_source_id, int level,
0159                   bool line_status)
0160 {
0161     struct kvm_lapic_irq irq;
0162     int r;
0163
0164     switch (e->type) {
0165     case KVM_IRQ_ROUTING_HV_SINT:
0166         return kvm_hv_set_sint(e, kvm, irq_source_id, level,
0167                        line_status);
0168
0169     case KVM_IRQ_ROUTING_MSI:
0170         if (kvm_msi_route_invalid(kvm, e))
0171             return -EINVAL;
0172
0173         kvm_set_msi_irq(kvm, e, &irq);
0174
0175         if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r, NULL))
0176             return r;
0177         break;
0178
0179 #ifdef CONFIG_KVM_XEN
0180     case KVM_IRQ_ROUTING_XEN_EVTCHN:
0181         if (!level)
0182             return -1;
0183
0184         return kvm_xen_set_evtchn_fast(&e->xen_evtchn, kvm);
0185 #endif
0186     default:
0187         break;
0188     }
0189
0190     return -EWOULDBLOCK;
0191 }
0192
0193 int kvm_request_irq_source_id(struct kvm *kvm)
0194 {
0195     unsigned long *bitmap = &kvm->arch.irq_sources_bitmap;
0196     int irq_source_id;
0197
0198     mutex_lock(&kvm->irq_lock);
0199     irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG);
0200
0201     if (irq_source_id >= BITS_PER_LONG) {
0202         printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources!\n");
0203         irq_source_id = -EFAULT;
0204         goto unlock;
0205     }
0206
0207     ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID);
0208     ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID);
0209     set_bit(irq_source_id, bitmap);
0210 unlock:
0211     mutex_unlock(&kvm->irq_lock);
0212
0213     return irq_source_id;
0214 }
0215
0216 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id)
0217 {
0218     ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID);
0219     ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID);
0220
0221     mutex_lock(&kvm->irq_lock);
0222     if (irq_source_id < 0 ||
0223         irq_source_id >= BITS_PER_LONG) {
0224         printk(KERN_ERR "kvm: IRQ source ID out of range!\n");
0225         goto unlock;
0226     }
0227     clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap);
0228     if (!irqchip_kernel(kvm))
0229         goto unlock;
0230
0231     kvm_ioapic_clear_all(kvm->arch.vioapic, irq_source_id);
0232     kvm_pic_clear_all(kvm->arch.vpic, irq_source_id);
0233 unlock:
0234     mutex_unlock(&kvm->irq_lock);
0235 }
0236
0237 void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
0238                     struct kvm_irq_mask_notifier *kimn)
0239 {
0240     mutex_lock(&kvm->irq_lock);
0241     kimn->irq = irq;
0242     hlist_add_head_rcu(&kimn->link, &kvm->arch.mask_notifier_list);
0243     mutex_unlock(&kvm->irq_lock);
0244 }
0245
0246 void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
0247                       struct kvm_irq_mask_notifier *kimn)
0248 {
0249     mutex_lock(&kvm->irq_lock);
0250     hlist_del_rcu(&kimn->link);
0251     mutex_unlock(&kvm->irq_lock);
0252     synchronize_srcu(&kvm->irq_srcu);
0253 }
0254
0255 void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin,
0256                  bool mask)
0257 {
0258     struct kvm_irq_mask_notifier *kimn;
0259     int idx, gsi;
0260
0261     idx = srcu_read_lock(&kvm->irq_srcu);
0262     gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
0263     if (gsi != -1)
0264         hlist_for_each_entry_rcu(kimn, &kvm->arch.mask_notifier_list, link)
0265             if (kimn->irq == gsi)
0266                 kimn->func(kimn, mask);
0267     srcu_read_unlock(&kvm->irq_srcu, idx);
0268 }
0269
0270 bool kvm_arch_can_set_irq_routing(struct kvm *kvm)
0271 {
0272     return irqchip_in_kernel(kvm);
0273 }
0274
0275 int kvm_set_routing_entry(struct kvm *kvm,
0276               struct kvm_kernel_irq_routing_entry *e,
0277               const struct kvm_irq_routing_entry *ue)
0278 {
0279     /* We can't check irqchip_in_kernel() here as some callers are
0280      * currently initializing the irqchip. Other callers should therefore
0281      * check kvm_arch_can_set_irq_routing() before calling this function.
0282      */
0283     switch (ue->type) {
0284     case KVM_IRQ_ROUTING_IRQCHIP:
0285         if (irqchip_split(kvm))
0286             return -EINVAL;
0287         e->irqchip.pin = ue->u.irqchip.pin;
0288         switch (ue->u.irqchip.irqchip) {
0289         case KVM_IRQCHIP_PIC_SLAVE:
0290             e->irqchip.pin += PIC_NUM_PINS / 2;
0291             fallthrough;
0292         case KVM_IRQCHIP_PIC_MASTER:
0293             if (ue->u.irqchip.pin >= PIC_NUM_PINS / 2)
0294                 return -EINVAL;
0295             e->set = kvm_set_pic_irq;
0296             break;
0297         case KVM_IRQCHIP_IOAPIC:
0298             if (ue->u.irqchip.pin >= KVM_IOAPIC_NUM_PINS)
0299                 return -EINVAL;
0300             e->set = kvm_set_ioapic_irq;
0301             break;
0302         default:
0303             return -EINVAL;
0304         }
0305         e->irqchip.irqchip = ue->u.irqchip.irqchip;
0306         break;
0307     case KVM_IRQ_ROUTING_MSI:
0308         e->set = kvm_set_msi;
0309         e->msi.address_lo = ue->u.msi.address_lo;
0310         e->msi.address_hi = ue->u.msi.address_hi;
0311         e->msi.data = ue->u.msi.data;
0312
0313         if (kvm_msi_route_invalid(kvm, e))
0314             return -EINVAL;
0315         break;
0316     case KVM_IRQ_ROUTING_HV_SINT:
0317         e->set = kvm_hv_set_sint;
0318         e->hv_sint.vcpu = ue->u.hv_sint.vcpu;
0319         e->hv_sint.sint = ue->u.hv_sint.sint;
0320         break;
0321 #ifdef CONFIG_KVM_XEN
0322     case KVM_IRQ_ROUTING_XEN_EVTCHN:
0323         return kvm_xen_setup_evtchn(kvm, e, ue);
0324 #endif
0325     default:
0326         return -EINVAL;
0327     }
0328
0329     return 0;
0330 }
0331
0332 bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq,
0333                  struct kvm_vcpu **dest_vcpu)
0334 {
0335     int r = 0;
0336     unsigned long i;
0337     struct kvm_vcpu *vcpu;
0338
0339     if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu))
0340         return true;
0341
0342     kvm_for_each_vcpu(i, vcpu, kvm) {
0343         if (!kvm_apic_present(vcpu))
0344             continue;
0345
0346         if (!kvm_apic_match_dest(vcpu, NULL, irq->shorthand,
0347                     irq->dest_id, irq->dest_mode))
0348             continue;
0349
0350         if (++r == 2)
0351             return false;
0352
0353         *dest_vcpu = vcpu;
0354     }
0355
0356     return r == 1;
0357 }
0358 EXPORT_SYMBOL_GPL(kvm_intr_is_single_vcpu);
0359
0360 #define IOAPIC_ROUTING_ENTRY(irq) \
0361     { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP,  \
0362       .u.irqchip = { .irqchip = KVM_IRQCHIP_IOAPIC, .pin = (irq) } }
0363 #define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq)
0364
0365 #define PIC_ROUTING_ENTRY(irq) \
0366     { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP,  \
0367       .u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } }
0368 #define ROUTING_ENTRY2(irq) \
0369     IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq)
0370
0371 static const struct kvm_irq_routing_entry default_routing[] = {
0372     ROUTING_ENTRY2(0), ROUTING_ENTRY2(1),
0373     ROUTING_ENTRY2(2), ROUTING_ENTRY2(3),
0374     ROUTING_ENTRY2(4), ROUTING_ENTRY2(5),
0375     ROUTING_ENTRY2(6), ROUTING_ENTRY2(7),
0376     ROUTING_ENTRY2(8), ROUTING_ENTRY2(9),
0377     ROUTING_ENTRY2(10), ROUTING_ENTRY2(11),
0378     ROUTING_ENTRY2(12), ROUTING_ENTRY2(13),
0379     ROUTING_ENTRY2(14), ROUTING_ENTRY2(15),
0380     ROUTING_ENTRY1(16), ROUTING_ENTRY1(17),
0381     ROUTING_ENTRY1(18), ROUTING_ENTRY1(19),
0382     ROUTING_ENTRY1(20), ROUTING_ENTRY1(21),
0383     ROUTING_ENTRY1(22), ROUTING_ENTRY1(23),
0384 };
0385
0386 int kvm_setup_default_irq_routing(struct kvm *kvm)
0387 {
0388     return kvm_set_irq_routing(kvm, default_routing,
0389                    ARRAY_SIZE(default_routing), 0);
0390 }
0391
0392 static const struct kvm_irq_routing_entry empty_routing[] = {};
0393
0394 int kvm_setup_empty_irq_routing(struct kvm *kvm)
0395 {
0396     return kvm_set_irq_routing(kvm, empty_routing, 0, 0);
0397 }
0398
0399 void kvm_arch_post_irq_routing_update(struct kvm *kvm)
0400 {
0401     if (!irqchip_split(kvm))
0402         return;
0403     kvm_make_scan_ioapic_request(kvm);
0404 }
0405
0406 void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu,
0407                 ulong *ioapic_handled_vectors)
0408 {
0409     struct kvm *kvm = vcpu->kvm;
0410     struct kvm_kernel_irq_routing_entry *entry;
0411     struct kvm_irq_routing_table *table;
0412     u32 i, nr_ioapic_pins;
0413     int idx;
0414
0415     idx = srcu_read_lock(&kvm->irq_srcu);
0416     table = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
0417     nr_ioapic_pins = min_t(u32, table->nr_rt_entries,
0418                    kvm->arch.nr_reserved_ioapic_pins);
0419     for (i = 0; i < nr_ioapic_pins; ++i) {
0420         hlist_for_each_entry(entry, &table->map[i], link) {
0421             struct kvm_lapic_irq irq;
0422
0423             if (entry->type != KVM_IRQ_ROUTING_MSI)
0424                 continue;
0425
0426             kvm_set_msi_irq(vcpu->kvm, entry, &irq);
0427
0428             if (irq.trig_mode &&
0429                 kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT,
0430                         irq.dest_id, irq.dest_mode))
0431                 __set_bit(irq.vector, ioapic_handled_vectors);
0432         }
0433     }
0434     srcu_read_unlock(&kvm->irq_srcu, idx);
0435 }
0436
0437 void kvm_arch_irq_routing_update(struct kvm *kvm)
0438 {
0439     kvm_hv_irq_routing_update(kvm);
0440 }