lib/aarch64/gic_v3.c

0001 // SPDX-License-Identifier: GPL-2.0
0002 /*
0003  * ARM Generic Interrupt Controller (GIC) v3 support
0004  */
0005
0006 #include <linux/sizes.h>
0007
0008 #include "kvm_util.h"
0009 #include "processor.h"
0010 #include "delay.h"
0011
0012 #include "gic_v3.h"
0013 #include "gic_private.h"
0014
0015 struct gicv3_data {
0016     void *dist_base;
0017     void *redist_base[GICV3_MAX_CPUS];
0018     unsigned int nr_cpus;
0019     unsigned int nr_spis;
0020 };
0021
0022 #define sgi_base_from_redist(redist_base)   (redist_base + SZ_64K)
0023 #define DIST_BIT                (1U << 31)
0024
0025 enum gicv3_intid_range {
0026     SGI_RANGE,
0027     PPI_RANGE,
0028     SPI_RANGE,
0029     INVALID_RANGE,
0030 };
0031
0032 static struct gicv3_data gicv3_data;
0033
0034 static void gicv3_gicd_wait_for_rwp(void)
0035 {
0036     unsigned int count = 100000; /* 1s */
0037
0038     while (readl(gicv3_data.dist_base + GICD_CTLR) & GICD_CTLR_RWP) {
0039         GUEST_ASSERT(count--);
0040         udelay(10);
0041     }
0042 }
0043
0044 static void gicv3_gicr_wait_for_rwp(void *redist_base)
0045 {
0046     unsigned int count = 100000; /* 1s */
0047
0048     while (readl(redist_base + GICR_CTLR) & GICR_CTLR_RWP) {
0049         GUEST_ASSERT(count--);
0050         udelay(10);
0051     }
0052 }
0053
0054 static void gicv3_wait_for_rwp(uint32_t cpu_or_dist)
0055 {
0056     if (cpu_or_dist & DIST_BIT)
0057         gicv3_gicd_wait_for_rwp();
0058     else
0059         gicv3_gicr_wait_for_rwp(gicv3_data.redist_base[cpu_or_dist]);
0060 }
0061
0062 static enum gicv3_intid_range get_intid_range(unsigned int intid)
0063 {
0064     switch (intid) {
0065     case 0 ... 15:
0066         return SGI_RANGE;
0067     case 16 ... 31:
0068         return PPI_RANGE;
0069     case 32 ... 1019:
0070         return SPI_RANGE;
0071     }
0072
0073     /* We should not be reaching here */
0074     GUEST_ASSERT(0);
0075
0076     return INVALID_RANGE;
0077 }
0078
0079 static uint64_t gicv3_read_iar(void)
0080 {
0081     uint64_t irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
0082
0083     dsb(sy);
0084     return irqstat;
0085 }
0086
0087 static void gicv3_write_eoir(uint32_t irq)
0088 {
0089     write_sysreg_s(irq, SYS_ICC_EOIR1_EL1);
0090     isb();
0091 }
0092
0093 static void gicv3_write_dir(uint32_t irq)
0094 {
0095     write_sysreg_s(irq, SYS_ICC_DIR_EL1);
0096     isb();
0097 }
0098
0099 static void gicv3_set_priority_mask(uint64_t mask)
0100 {
0101     write_sysreg_s(mask, SYS_ICC_PMR_EL1);
0102 }
0103
0104 static void gicv3_set_eoi_split(bool split)
0105 {
0106     uint32_t val;
0107
0108     /*
0109      * All other fields are read-only, so no need to read CTLR first. In
0110      * fact, the kernel does the same.
0111      */
0112     val = split ? (1U << 1) : 0;
0113     write_sysreg_s(val, SYS_ICC_CTLR_EL1);
0114     isb();
0115 }
0116
0117 uint32_t gicv3_reg_readl(uint32_t cpu_or_dist, uint64_t offset)
0118 {
0119     void *base = cpu_or_dist & DIST_BIT ? gicv3_data.dist_base
0120         : sgi_base_from_redist(gicv3_data.redist_base[cpu_or_dist]);
0121     return readl(base + offset);
0122 }
0123
0124 void gicv3_reg_writel(uint32_t cpu_or_dist, uint64_t offset, uint32_t reg_val)
0125 {
0126     void *base = cpu_or_dist & DIST_BIT ? gicv3_data.dist_base
0127         : sgi_base_from_redist(gicv3_data.redist_base[cpu_or_dist]);
0128     writel(reg_val, base + offset);
0129 }
0130
0131 uint32_t gicv3_getl_fields(uint32_t cpu_or_dist, uint64_t offset, uint32_t mask)
0132 {
0133     return gicv3_reg_readl(cpu_or_dist, offset) & mask;
0134 }
0135
0136 void gicv3_setl_fields(uint32_t cpu_or_dist, uint64_t offset,
0137         uint32_t mask, uint32_t reg_val)
0138 {
0139     uint32_t tmp = gicv3_reg_readl(cpu_or_dist, offset) & ~mask;
0140
0141     tmp |= (reg_val & mask);
0142     gicv3_reg_writel(cpu_or_dist, offset, tmp);
0143 }
0144
0145 /*
0146  * We use a single offset for the distributor and redistributor maps as they
0147  * have the same value in both. The only exceptions are registers that only
0148  * exist in one and not the other, like GICR_WAKER that doesn't exist in the
0149  * distributor map. Such registers are conveniently marked as reserved in the
0150  * map that doesn't implement it; like GICR_WAKER's offset of 0x0014 being
0151  * marked as "Reserved" in the Distributor map.
0152  */
0153 static void gicv3_access_reg(uint32_t intid, uint64_t offset,
0154         uint32_t reg_bits, uint32_t bits_per_field,
0155         bool write, uint32_t *val)
0156 {
0157     uint32_t cpu = guest_get_vcpuid();
0158     enum gicv3_intid_range intid_range = get_intid_range(intid);
0159     uint32_t fields_per_reg, index, mask, shift;
0160     uint32_t cpu_or_dist;
0161
0162     GUEST_ASSERT(bits_per_field <= reg_bits);
0163     GUEST_ASSERT(!write || *val < (1U << bits_per_field));
0164     /*
0165      * This function does not support 64 bit accesses. Just asserting here
0166      * until we implement readq/writeq.
0167      */
0168     GUEST_ASSERT(reg_bits == 32);
0169
0170     fields_per_reg = reg_bits / bits_per_field;
0171     index = intid % fields_per_reg;
0172     shift = index * bits_per_field;
0173     mask = ((1U << bits_per_field) - 1) << shift;
0174
0175     /* Set offset to the actual register holding intid's config. */
0176     offset += (intid / fields_per_reg) * (reg_bits / 8);
0177
0178     cpu_or_dist = (intid_range == SPI_RANGE) ? DIST_BIT : cpu;
0179
0180     if (write)
0181         gicv3_setl_fields(cpu_or_dist, offset, mask, *val << shift);
0182     *val = gicv3_getl_fields(cpu_or_dist, offset, mask) >> shift;
0183 }
0184
0185 static void gicv3_write_reg(uint32_t intid, uint64_t offset,
0186         uint32_t reg_bits, uint32_t bits_per_field, uint32_t val)
0187 {
0188     gicv3_access_reg(intid, offset, reg_bits,
0189             bits_per_field, true, &val);
0190 }
0191
0192 static uint32_t gicv3_read_reg(uint32_t intid, uint64_t offset,
0193         uint32_t reg_bits, uint32_t bits_per_field)
0194 {
0195     uint32_t val;
0196
0197     gicv3_access_reg(intid, offset, reg_bits,
0198             bits_per_field, false, &val);
0199     return val;
0200 }
0201
0202 static void gicv3_set_priority(uint32_t intid, uint32_t prio)
0203 {
0204     gicv3_write_reg(intid, GICD_IPRIORITYR, 32, 8, prio);
0205 }
0206
0207 /* Sets the intid to be level-sensitive or edge-triggered. */
0208 static void gicv3_irq_set_config(uint32_t intid, bool is_edge)
0209 {
0210     uint32_t val;
0211
0212     /* N/A for private interrupts. */
0213     GUEST_ASSERT(get_intid_range(intid) == SPI_RANGE);
0214     val = is_edge ? 2 : 0;
0215     gicv3_write_reg(intid, GICD_ICFGR, 32, 2, val);
0216 }
0217
0218 static void gicv3_irq_enable(uint32_t intid)
0219 {
0220     bool is_spi = get_intid_range(intid) == SPI_RANGE;
0221     uint32_t cpu = guest_get_vcpuid();
0222
0223     gicv3_write_reg(intid, GICD_ISENABLER, 32, 1, 1);
0224     gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
0225 }
0226
0227 static void gicv3_irq_disable(uint32_t intid)
0228 {
0229     bool is_spi = get_intid_range(intid) == SPI_RANGE;
0230     uint32_t cpu = guest_get_vcpuid();
0231
0232     gicv3_write_reg(intid, GICD_ICENABLER, 32, 1, 1);
0233     gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
0234 }
0235
0236 static void gicv3_irq_set_active(uint32_t intid)
0237 {
0238     gicv3_write_reg(intid, GICD_ISACTIVER, 32, 1, 1);
0239 }
0240
0241 static void gicv3_irq_clear_active(uint32_t intid)
0242 {
0243     gicv3_write_reg(intid, GICD_ICACTIVER, 32, 1, 1);
0244 }
0245
0246 static bool gicv3_irq_get_active(uint32_t intid)
0247 {
0248     return gicv3_read_reg(intid, GICD_ISACTIVER, 32, 1);
0249 }
0250
0251 static void gicv3_irq_set_pending(uint32_t intid)
0252 {
0253     gicv3_write_reg(intid, GICD_ISPENDR, 32, 1, 1);
0254 }
0255
0256 static void gicv3_irq_clear_pending(uint32_t intid)
0257 {
0258     gicv3_write_reg(intid, GICD_ICPENDR, 32, 1, 1);
0259 }
0260
0261 static bool gicv3_irq_get_pending(uint32_t intid)
0262 {
0263     return gicv3_read_reg(intid, GICD_ISPENDR, 32, 1);
0264 }
0265
0266 static void gicv3_enable_redist(void *redist_base)
0267 {
0268     uint32_t val = readl(redist_base + GICR_WAKER);
0269     unsigned int count = 100000; /* 1s */
0270
0271     val &= ~GICR_WAKER_ProcessorSleep;
0272     writel(val, redist_base + GICR_WAKER);
0273
0274     /* Wait until the processor is 'active' */
0275     while (readl(redist_base + GICR_WAKER) & GICR_WAKER_ChildrenAsleep) {
0276         GUEST_ASSERT(count--);
0277         udelay(10);
0278     }
0279 }
0280
0281 static inline void *gicr_base_cpu(void *redist_base, uint32_t cpu)
0282 {
0283     /* Align all the redistributors sequentially */
0284     return redist_base + cpu * SZ_64K * 2;
0285 }
0286
0287 static void gicv3_cpu_init(unsigned int cpu, void *redist_base)
0288 {
0289     void *sgi_base;
0290     unsigned int i;
0291     void *redist_base_cpu;
0292
0293     GUEST_ASSERT(cpu < gicv3_data.nr_cpus);
0294
0295     redist_base_cpu = gicr_base_cpu(redist_base, cpu);
0296     sgi_base = sgi_base_from_redist(redist_base_cpu);
0297
0298     gicv3_enable_redist(redist_base_cpu);
0299
0300     /*
0301      * Mark all the SGI and PPI interrupts as non-secure Group-1.
0302      * Also, deactivate and disable them.
0303      */
0304     writel(~0, sgi_base + GICR_IGROUPR0);
0305     writel(~0, sgi_base + GICR_ICACTIVER0);
0306     writel(~0, sgi_base + GICR_ICENABLER0);
0307
0308     /* Set a default priority for all the SGIs and PPIs */
0309     for (i = 0; i < 32; i += 4)
0310         writel(GICD_INT_DEF_PRI_X4,
0311                 sgi_base + GICR_IPRIORITYR0 + i);
0312
0313     gicv3_gicr_wait_for_rwp(redist_base_cpu);
0314
0315     /* Enable the GIC system register (ICC_*) access */
0316     write_sysreg_s(read_sysreg_s(SYS_ICC_SRE_EL1) | ICC_SRE_EL1_SRE,
0317             SYS_ICC_SRE_EL1);
0318
0319     /* Set a default priority threshold */
0320     write_sysreg_s(ICC_PMR_DEF_PRIO, SYS_ICC_PMR_EL1);
0321
0322     /* Enable non-secure Group-1 interrupts */
0323     write_sysreg_s(ICC_IGRPEN1_EL1_ENABLE, SYS_ICC_GRPEN1_EL1);
0324
0325     gicv3_data.redist_base[cpu] = redist_base_cpu;
0326 }
0327
0328 static void gicv3_dist_init(void)
0329 {
0330     void *dist_base = gicv3_data.dist_base;
0331     unsigned int i;
0332
0333     /* Disable the distributor until we set things up */
0334     writel(0, dist_base + GICD_CTLR);
0335     gicv3_gicd_wait_for_rwp();
0336
0337     /*
0338      * Mark all the SPI interrupts as non-secure Group-1.
0339      * Also, deactivate and disable them.
0340      */
0341     for (i = 32; i < gicv3_data.nr_spis; i += 32) {
0342         writel(~0, dist_base + GICD_IGROUPR + i / 8);
0343         writel(~0, dist_base + GICD_ICACTIVER + i / 8);
0344         writel(~0, dist_base + GICD_ICENABLER + i / 8);
0345     }
0346
0347     /* Set a default priority for all the SPIs */
0348     for (i = 32; i < gicv3_data.nr_spis; i += 4)
0349         writel(GICD_INT_DEF_PRI_X4,
0350                 dist_base + GICD_IPRIORITYR + i);
0351
0352     /* Wait for the settings to sync-in */
0353     gicv3_gicd_wait_for_rwp();
0354
0355     /* Finally, enable the distributor globally with ARE */
0356     writel(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A |
0357             GICD_CTLR_ENABLE_G1, dist_base + GICD_CTLR);
0358     gicv3_gicd_wait_for_rwp();
0359 }
0360
0361 static void gicv3_init(unsigned int nr_cpus, void *dist_base)
0362 {
0363     GUEST_ASSERT(nr_cpus <= GICV3_MAX_CPUS);
0364
0365     gicv3_data.nr_cpus = nr_cpus;
0366     gicv3_data.dist_base = dist_base;
0367     gicv3_data.nr_spis = GICD_TYPER_SPIS(
0368                 readl(gicv3_data.dist_base + GICD_TYPER));
0369     if (gicv3_data.nr_spis > 1020)
0370         gicv3_data.nr_spis = 1020;
0371
0372     /*
0373      * Initialize only the distributor for now.
0374      * The redistributor and CPU interfaces are initialized
0375      * later for every PE.
0376      */
0377     gicv3_dist_init();
0378 }
0379
0380 const struct gic_common_ops gicv3_ops = {
0381     .gic_init = gicv3_init,
0382     .gic_cpu_init = gicv3_cpu_init,
0383     .gic_irq_enable = gicv3_irq_enable,
0384     .gic_irq_disable = gicv3_irq_disable,
0385     .gic_read_iar = gicv3_read_iar,
0386     .gic_write_eoir = gicv3_write_eoir,
0387     .gic_write_dir = gicv3_write_dir,
0388     .gic_set_priority_mask = gicv3_set_priority_mask,
0389     .gic_set_eoi_split = gicv3_set_eoi_split,
0390     .gic_set_priority = gicv3_set_priority,
0391     .gic_irq_set_active = gicv3_irq_set_active,
0392     .gic_irq_clear_active = gicv3_irq_clear_active,
0393     .gic_irq_get_active = gicv3_irq_get_active,
0394     .gic_irq_set_pending = gicv3_irq_set_pending,
0395     .gic_irq_clear_pending = gicv3_irq_clear_pending,
0396     .gic_irq_get_pending = gicv3_irq_get_pending,
0397     .gic_irq_set_config = gicv3_irq_set_config,
0398 };