drivers/irqchip/irq-gic-v4.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
0004  * Author: Marc Zyngier <marc.zyngier@arm.com>
0005  */
0006
0007 #include <linux/interrupt.h>
0008 #include <linux/irq.h>
0009 #include <linux/irqdomain.h>
0010 #include <linux/msi.h>
0011 #include <linux/sched.h>
0012
0013 #include <linux/irqchip/arm-gic-v4.h>
0014
0015 /*
0016  * WARNING: The blurb below assumes that you understand the
0017  * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
0018  * translated into GICv4 commands. So it effectively targets at most
0019  * two individuals. You know who you are.
0020  *
0021  * The core GICv4 code is designed to *avoid* exposing too much of the
0022  * core GIC code (that would in turn leak into the hypervisor code),
0023  * and instead provide a hypervisor agnostic interface to the HW (of
0024  * course, the astute reader will quickly realize that hypervisor
0025  * agnostic actually means KVM-specific - what were you thinking?).
0026  *
0027  * In order to achieve a modicum of isolation, we try to hide most of
0028  * the GICv4 "stuff" behind normal irqchip operations:
0029  *
0030  * - Any guest-visible VLPI is backed by a Linux interrupt (and a
0031  *   physical LPI which gets unmapped when the guest maps the
0032  *   VLPI). This allows the same DevID/EventID pair to be either
0033  *   mapped to the LPI (host) or the VLPI (guest). Note that this is
0034  *   exclusive, and you cannot have both.
0035  *
0036  * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
0037  *
0038  * - Guest INT/CLEAR commands are implemented through
0039  *   irq_set_irqchip_state().
0040  *
0041  * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
0042  *   issuing an INV after changing a priority) gets shoved into the
0043  *   irq_set_vcpu_affinity() method. While this is quite horrible
0044  *   (let's face it, this is the irqchip version of an ioctl), it
0045  *   confines the crap to a single location. And map/unmap really is
0046  *   about setting the affinity of a VLPI to a vcpu, so only INV is
0047  *   majorly out of place. So there.
0048  *
0049  * A number of commands are simply not provided by this interface, as
0050  * they do not make direct sense. For example, MAPD is purely local to
0051  * the virtual ITS (because it references a virtual device, and the
0052  * physical ITS is still very much in charge of the physical
0053  * device). Same goes for things like MAPC (the physical ITS deals
0054  * with the actual vPE affinity, and not the braindead concept of
0055  * collection). SYNC is not provided either, as each and every command
0056  * is followed by a VSYNC. This could be relaxed in the future, should
0057  * this be seen as a bottleneck (yes, this means *never*).
0058  *
0059  * But handling VLPIs is only one side of the job of the GICv4
0060  * code. The other (darker) side is to take care of the doorbell
0061  * interrupts which are delivered when a VLPI targeting a non-running
0062  * vcpu is being made pending.
0063  *
0064  * The choice made here is that each vcpu (VPE in old northern GICv4
0065  * dialect) gets a single doorbell LPI, no matter how many interrupts
0066  * are targeting it. This has a nice property, which is that the
0067  * interrupt becomes a handle for the VPE, and that the hypervisor
0068  * code can manipulate it through the normal interrupt API:
0069  *
0070  * - VMs (or rather the VM abstraction that matters to the GIC)
0071  *   contain an irq domain where each interrupt maps to a VPE. In
0072  *   turn, this domain sits on top of the normal LPI allocator, and a
0073  *   specially crafted irq_chip implementation.
0074  *
0075  * - mask/unmask do what is expected on the doorbell interrupt.
0076  *
0077  * - irq_set_affinity is used to move a VPE from one redistributor to
0078  *   another.
0079  *
0080  * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
0081  *   creating a new sub-API, namely scheduling/descheduling a VPE
0082  *   (which involves programming GICR_V{PROP,PEND}BASER) and
0083  *   performing INVALL operations.
0084  */
0085
0086 static struct irq_domain *gic_domain;
0087 static const struct irq_domain_ops *vpe_domain_ops;
0088 static const struct irq_domain_ops *sgi_domain_ops;
0089
0090 #ifdef CONFIG_ARM64
0091 #include <asm/cpufeature.h>
0092
0093 bool gic_cpuif_has_vsgi(void)
0094 {
0095     unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
0096
0097     fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_GIC_SHIFT);
0098
0099     return fld >= 0x3;
0100 }
0101 #else
0102 bool gic_cpuif_has_vsgi(void)
0103 {
0104     return false;
0105 }
0106 #endif
0107
0108 static bool has_v4_1(void)
0109 {
0110     return !!sgi_domain_ops;
0111 }
0112
0113 static bool has_v4_1_sgi(void)
0114 {
0115     return has_v4_1() && gic_cpuif_has_vsgi();
0116 }
0117
0118 static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
0119 {
0120     char *name;
0121     int sgi_base;
0122
0123     if (!has_v4_1_sgi())
0124         return 0;
0125
0126     name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current));
0127     if (!name)
0128         goto err;
0129
0130     vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx);
0131     if (!vpe->fwnode)
0132         goto err;
0133
0134     kfree(name);
0135     name = NULL;
0136
0137     vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16,
0138                            sgi_domain_ops, vpe);
0139     if (!vpe->sgi_domain)
0140         goto err;
0141
0142     sgi_base = __irq_domain_alloc_irqs(vpe->sgi_domain, -1, 16,
0143                            NUMA_NO_NODE, vpe,
0144                            false, NULL);
0145     if (sgi_base <= 0)
0146         goto err;
0147
0148     return 0;
0149
0150 err:
0151     if (vpe->sgi_domain)
0152         irq_domain_remove(vpe->sgi_domain);
0153     if (vpe->fwnode)
0154         irq_domain_free_fwnode(vpe->fwnode);
0155     kfree(name);
0156     return -ENOMEM;
0157 }
0158
0159 int its_alloc_vcpu_irqs(struct its_vm *vm)
0160 {
0161     int vpe_base_irq, i;
0162
0163     vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe",
0164                               task_pid_nr(current));
0165     if (!vm->fwnode)
0166         goto err;
0167
0168     vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes,
0169                          vm->fwnode, vpe_domain_ops,
0170                          vm);
0171     if (!vm->domain)
0172         goto err;
0173
0174     for (i = 0; i < vm->nr_vpes; i++) {
0175         vm->vpes[i]->its_vm = vm;
0176         vm->vpes[i]->idai = true;
0177     }
0178
0179     vpe_base_irq = __irq_domain_alloc_irqs(vm->domain, -1, vm->nr_vpes,
0180                            NUMA_NO_NODE, vm,
0181                            false, NULL);
0182     if (vpe_base_irq <= 0)
0183         goto err;
0184
0185     for (i = 0; i < vm->nr_vpes; i++) {
0186         int ret;
0187         vm->vpes[i]->irq = vpe_base_irq + i;
0188         ret = its_alloc_vcpu_sgis(vm->vpes[i], i);
0189         if (ret)
0190             goto err;
0191     }
0192
0193     return 0;
0194
0195 err:
0196     if (vm->domain)
0197         irq_domain_remove(vm->domain);
0198     if (vm->fwnode)
0199         irq_domain_free_fwnode(vm->fwnode);
0200
0201     return -ENOMEM;
0202 }
0203
0204 static void its_free_sgi_irqs(struct its_vm *vm)
0205 {
0206     int i;
0207
0208     if (!has_v4_1_sgi())
0209         return;
0210
0211     for (i = 0; i < vm->nr_vpes; i++) {
0212         unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0);
0213
0214         if (WARN_ON(!irq))
0215             continue;
0216
0217         irq_domain_free_irqs(irq, 16);
0218         irq_domain_remove(vm->vpes[i]->sgi_domain);
0219         irq_domain_free_fwnode(vm->vpes[i]->fwnode);
0220     }
0221 }
0222
0223 void its_free_vcpu_irqs(struct its_vm *vm)
0224 {
0225     its_free_sgi_irqs(vm);
0226     irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes);
0227     irq_domain_remove(vm->domain);
0228     irq_domain_free_fwnode(vm->fwnode);
0229 }
0230
0231 static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
0232 {
0233     return irq_set_vcpu_affinity(vpe->irq, info);
0234 }
0235
0236 int its_make_vpe_non_resident(struct its_vpe *vpe, bool db)
0237 {
0238     struct irq_desc *desc = irq_to_desc(vpe->irq);
0239     struct its_cmd_info info = { };
0240     int ret;
0241
0242     WARN_ON(preemptible());
0243
0244     info.cmd_type = DESCHEDULE_VPE;
0245     if (has_v4_1()) {
0246         /* GICv4.1 can directly deal with doorbells */
0247         info.req_db = db;
0248     } else {
0249         /* Undo the nested disable_irq() calls... */
0250         while (db && irqd_irq_disabled(&desc->irq_data))
0251             enable_irq(vpe->irq);
0252     }
0253
0254     ret = its_send_vpe_cmd(vpe, &info);
0255     if (!ret)
0256         vpe->resident = false;
0257
0258     vpe->ready = false;
0259
0260     return ret;
0261 }
0262
0263 int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en)
0264 {
0265     struct its_cmd_info info = { };
0266     int ret;
0267
0268     WARN_ON(preemptible());
0269
0270     info.cmd_type = SCHEDULE_VPE;
0271     if (has_v4_1()) {
0272         info.g0en = g0en;
0273         info.g1en = g1en;
0274     } else {
0275         /* Disabled the doorbell, as we're about to enter the guest */
0276         disable_irq_nosync(vpe->irq);
0277     }
0278
0279     ret = its_send_vpe_cmd(vpe, &info);
0280     if (!ret)
0281         vpe->resident = true;
0282
0283     return ret;
0284 }
0285
0286 int its_commit_vpe(struct its_vpe *vpe)
0287 {
0288     struct its_cmd_info info = {
0289         .cmd_type = COMMIT_VPE,
0290     };
0291     int ret;
0292
0293     WARN_ON(preemptible());
0294
0295     ret = its_send_vpe_cmd(vpe, &info);
0296     if (!ret)
0297         vpe->ready = true;
0298
0299     return ret;
0300 }
0301
0302
0303 int its_invall_vpe(struct its_vpe *vpe)
0304 {
0305     struct its_cmd_info info = {
0306         .cmd_type = INVALL_VPE,
0307     };
0308
0309     return its_send_vpe_cmd(vpe, &info);
0310 }
0311
0312 int its_map_vlpi(int irq, struct its_vlpi_map *map)
0313 {
0314     struct its_cmd_info info = {
0315         .cmd_type = MAP_VLPI,
0316         {
0317             .map      = map,
0318         },
0319     };
0320     int ret;
0321
0322     /*
0323      * The host will never see that interrupt firing again, so it
0324      * is vital that we don't do any lazy masking.
0325      */
0326     irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY);
0327
0328     ret = irq_set_vcpu_affinity(irq, &info);
0329     if (ret)
0330         irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
0331
0332     return ret;
0333 }
0334
0335 int its_get_vlpi(int irq, struct its_vlpi_map *map)
0336 {
0337     struct its_cmd_info info = {
0338         .cmd_type = GET_VLPI,
0339         {
0340             .map      = map,
0341         },
0342     };
0343
0344     return irq_set_vcpu_affinity(irq, &info);
0345 }
0346
0347 int its_unmap_vlpi(int irq)
0348 {
0349     irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY);
0350     return irq_set_vcpu_affinity(irq, NULL);
0351 }
0352
0353 int its_prop_update_vlpi(int irq, u8 config, bool inv)
0354 {
0355     struct its_cmd_info info = {
0356         .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
0357         {
0358             .config   = config,
0359         },
0360     };
0361
0362     return irq_set_vcpu_affinity(irq, &info);
0363 }
0364
0365 int its_prop_update_vsgi(int irq, u8 priority, bool group)
0366 {
0367     struct its_cmd_info info = {
0368         .cmd_type = PROP_UPDATE_VSGI,
0369         {
0370             .priority   = priority,
0371             .group      = group,
0372         },
0373     };
0374
0375     return irq_set_vcpu_affinity(irq, &info);
0376 }
0377
0378 int its_init_v4(struct irq_domain *domain,
0379         const struct irq_domain_ops *vpe_ops,
0380         const struct irq_domain_ops *sgi_ops)
0381 {
0382     if (domain) {
0383         pr_info("ITS: Enabling GICv4 support\n");
0384         gic_domain = domain;
0385         vpe_domain_ops = vpe_ops;
0386         sgi_domain_ops = sgi_ops;
0387         return 0;
0388     }
0389
0390     pr_err("ITS: No GICv4 VPE domain allocated\n");
0391     return -ENODEV;
0392 }