개념 설명 전체 · v6.18.37 / drivers/irqchip/irq-gic-v3.c
1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #define pr_fmt(fmt) "GICv3: " fmt 8 9 #include <linux/acpi.h> 10 #include <linux/cpu.h> 11 #include <linux/cpu_pm.h> 12 #include <linux/delay.h> 13 #include <linux/interrupt.h> 14 #include <linux/irqdomain.h> 15 #include <linux/kernel.h> 16 #include <linux/kstrtox.h> 17 #include <linux/of.h> 18 #include <linux/of_address.h> 19 #include <linux/of_irq.h> 20 #include <linux/percpu.h> 21 #include <linux/refcount.h> 22 #include <linux/slab.h> 23 #include <linux/iopoll.h> 24 25 #include <linux/irqchip.h> 26 #include <linux/irqchip/arm-gic-common.h> 27 #include <linux/irqchip/arm-gic-v3.h> 28 #include <linux/irqchip/arm-gic-v3-prio.h> 29 #include <linux/irqchip/irq-partition-percpu.h> 30 #include <linux/bitfield.h> 31 #include <linux/bits.h> 32 #include <linux/arm-smccc.h> 33 34 #include <asm/cputype.h> 35 #include <asm/exception.h> 36 #include <asm/smp_plat.h> 37 #include <asm/virt.h> 38 39 #include "irq-gic-common.h" 40 41 static u8 dist_prio_irq __ro_after_init = GICV3_PRIO_IRQ; 42 static u8 dist_prio_nmi __ro_after_init = GICV3_PRIO_NMI; 43 44 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 45 #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) 46 #define FLAGS_WORKAROUND_ASR_ERRATUM_8601001 (1ULL << 2) 47 #define FLAGS_WORKAROUND_INSECURE (1ULL << 3) 48 49 #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 50 51 static struct cpumask broken_rdists __read_mostly __maybe_unused; 52 53 struct redist_region { 54 void __iomem *redist_base; 55 phys_addr_t phys_base; 56 bool single_redist; 57 }; 58 59 struct gic_chip_data { 60 struct fwnode_handle *fwnode; 61 phys_addr_t dist_phys_base; 62 void __iomem *dist_base; 63 struct redist_region *redist_regions; 64 struct rdists rdists; 65 struct irq_domain *domain; 66 u64 redist_stride; 67 u32 nr_redist_regions; 68 u64 flags; 69 bool has_rss; 70 unsigned int ppi_nr; 71 struct partition_desc **ppi_descs; 72 }; 73 74 #define T241_CHIPS_MAX 4 75 static void __iomem *t241_dist_base_alias[T241_CHIPS_MAX] __read_mostly; 76 static DEFINE_STATIC_KEY_FALSE(gic_nvidia_t241_erratum); 77 78 static DEFINE_STATIC_KEY_FALSE(gic_arm64_2941627_erratum); 79 80 static struct gic_chip_data gic_data __read_mostly; 81 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 82 83 #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 84 #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) 85 #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) 86 87 static bool nmi_support_forbidden; 88 89 /* 90 * There are 16 SGIs, though we only actually use 8 in Linux. The other 8 SGIs 91 * are potentially stolen by the secure side. Some code, especially code dealing 92 * with hwirq IDs, is simplified by accounting for all 16. 93 */ 94 #define SGI_NR 16 95 96 /* 97 * The behaviours of RPR and PMR registers differ depending on the value of 98 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 99 * distributor and redistributors depends on whether security is enabled in the 100 * GIC. 101 * 102 * When security is enabled, non-secure priority values from the (re)distributor 103 * are presented to the GIC CPUIF as follow: 104 * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 105 * 106 * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure 107 * EL1 are subject to a similar operation thus matching the priorities presented 108 * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0, 109 * these values are unchanged by the GIC. 110 * 111 * see GICv3/GICv4 Architecture Specification (IHI0069D): 112 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 113 * priorities. 114 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 115 * interrupt. 116 */ 117 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 118 119 static u32 gic_get_pribits(void) 120 { 121 u32 pribits; 122 123 pribits = gic_read_ctlr(); 124 pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 125 pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 126 pribits++; 127 128 return pribits; 129 } 130 131 static bool gic_has_group0(void) 132 { 133 u32 val; 134 u32 old_pmr; 135 136 old_pmr = gic_read_pmr(); 137 138 /* 139 * Let's find out if Group0 is under control of EL3 or not by 140 * setting the highest possible, non-zero priority in PMR. 141 * 142 * If SCR_EL3.FIQ is set, the priority gets shifted down in 143 * order for the CPU interface to set bit 7, and keep the 144 * actual priority in the non-secure range. In the process, it 145 * looses the least significant bit and the actual priority 146 * becomes 0x80. Reading it back returns 0, indicating that 147 * we're don't have access to Group0. 148 */ 149 gic_write_pmr(BIT(8 - gic_get_pribits())); 150 val = gic_read_pmr(); 151 152 gic_write_pmr(old_pmr); 153 154 return val != 0; 155 } 156 157 static inline bool gic_dist_security_disabled(void) 158 { 159 return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 160 } 161 162 static bool cpus_have_security_disabled __ro_after_init; 163 static bool cpus_have_group0 __ro_after_init; 164 165 static void __init gic_prio_init(void) 166 { 167 bool ds; 168 169 cpus_have_group0 = gic_has_group0(); 170 171 ds = gic_dist_security_disabled(); 172 if ((gic_data.flags & FLAGS_WORKAROUND_INSECURE) && !ds) { 173 if (cpus_have_group0) { 174 u32 val; 175 176 val = readl_relaxed(gic_data.dist_base + GICD_CTLR); 177 val |= GICD_CTLR_DS; 178 writel_relaxed(val, gic_data.dist_base + GICD_CTLR); 179 180 ds = gic_dist_security_disabled(); 181 if (ds) 182 pr_warn("Broken GIC integration, security disabled\n"); 183 } else { 184 pr_warn("Broken GIC integration, pNMI forbidden\n"); 185 nmi_support_forbidden = true; 186 } 187 } 188 189 cpus_have_security_disabled = ds; 190 191 /* 192 * How priority values are used by the GIC depends on two things: 193 * the security state of the GIC (controlled by the GICD_CTLR.DS bit) 194 * and if Group 0 interrupts can be delivered to Linux in the non-secure 195 * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the 196 * way priorities are presented in ICC_PMR_EL1 and in the distributor: 197 * 198 * GICD_CTLR.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Distributor 199 * ------------------------------------------------------- 200 * 1 | - | unchanged | unchanged 201 * ------------------------------------------------------- 202 * 0 | 1 | non-secure | non-secure 203 * ------------------------------------------------------- 204 * 0 | 0 | unchanged | non-secure 205 * 206 * In the non-secure view reads and writes are modified: 207 * 208 * - A value written is right-shifted by one and the MSB is set, 209 * forcing the priority into the non-secure range. 210 * 211 * - A value read is left-shifted by one. 212 * 213 * In the first two cases, where ICC_PMR_EL1 and the interrupt priority 214 * are both either modified or unchanged, we can use the same set of 215 * priorities. 216 * 217 * In the last case, where only the interrupt priorities are modified to 218 * be in the non-secure range, we program the non-secure values into 219 * the distributor to match the PMR values we want. 220 */ 221 if (cpus_have_group0 && !cpus_have_security_disabled) { 222 dist_prio_irq = __gicv3_prio_to_ns(dist_prio_irq); 223 dist_prio_nmi = __gicv3_prio_to_ns(dist_prio_nmi); 224 } 225 226 pr_info("GICD_CTLR.DS=%d, SCR_EL3.FIQ=%d\n", 227 cpus_have_security_disabled, 228 !cpus_have_group0); 229 } 230 231 /* rdist_nmi_refs[n] == number of cpus having the rdist interrupt n set as NMI */ 232 static refcount_t *rdist_nmi_refs; 233 234 static struct gic_kvm_info gic_v3_kvm_info __initdata; 235 static DEFINE_PER_CPU(bool, has_rss); 236 237 #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 238 #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 239 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 240 #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 241 242 /* Our default, arbitrary priority value. Linux only uses one anyway. */ 243 #define DEFAULT_PMR_VALUE 0xf0 244 245 enum gic_intid_range { 246 SGI_RANGE, 247 PPI_RANGE, 248 SPI_RANGE, 249 EPPI_RANGE, 250 ESPI_RANGE, 251 LPI_RANGE, 252 __INVALID_RANGE__ 253 }; 254 255 static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) 256 { 257 switch (hwirq) { 258 case 0 ... 15: 259 return SGI_RANGE; 260 case 16 ... 31: 261 return PPI_RANGE; 262 case 32 ... 1019: 263 return SPI_RANGE; 264 case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): 265 return EPPI_RANGE; 266 case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): 267 return ESPI_RANGE; 268 case 8192 ... GENMASK(23, 0): 269 return LPI_RANGE; 270 default: 271 return __INVALID_RANGE__; 272 } 273 } 274 275 static enum gic_intid_range get_intid_range(struct irq_data *d) 276 { 277 return __get_intid_range(d->hwirq); 278 } 279 280 static inline bool gic_irq_in_rdist(struct irq_data *d) 281 { 282 switch (get_intid_range(d)) { 283 case SGI_RANGE: 284 case PPI_RANGE: 285 case EPPI_RANGE: 286 return true; 287 default: 288 return false; 289 } 290 } 291 292 static inline void __iomem *gic_dist_base_alias(struct irq_data *d) 293 { 294 if (static_branch_unlikely(&gic_nvidia_t241_erratum)) { 295 irq_hw_number_t hwirq = irqd_to_hwirq(d); 296 u32 chip; 297 298 /* 299 * For the erratum T241-FABRIC-4, read accesses to GICD_In{E} 300 * registers are directed to the chip that owns the SPI. The 301 * the alias region can also be used for writes to the 302 * GICD_In{E} except GICD_ICENABLERn. Each chip has support 303 * for 320 {E}SPIs. Mappings for all 4 chips: 304 * Chip0 = 32-351 305 * Chip1 = 352-671 306 * Chip2 = 672-991 307 * Chip3 = 4096-4415 308 */ 309 switch (__get_intid_range(hwirq)) { 310 case SPI_RANGE: 311 chip = (hwirq - 32) / 320; 312 break; 313 case ESPI_RANGE: 314 chip = 3; 315 break; 316 default: 317 unreachable(); 318 } 319 return t241_dist_base_alias[chip]; 320 } 321 322 return gic_data.dist_base; 323 } 324 325 static inline void __iomem *gic_dist_base(struct irq_data *d) 326 { 327 switch (get_intid_range(d)) { 328 case SGI_RANGE: 329 case PPI_RANGE: 330 case EPPI_RANGE: 331 /* SGI+PPI -> SGI_base for this CPU */ 332 return gic_data_rdist_sgi_base(); 333 334 case SPI_RANGE: 335 case ESPI_RANGE: 336 /* SPI -> dist_base */ 337 return gic_data.dist_base; 338 339 default: 340 return NULL; 341 } 342 } 343 344 static void gic_do_wait_for_rwp(void __iomem *base, u32 bit) 345 { 346 u32 val; 347 int ret; 348 349 ret = readl_relaxed_poll_timeout_atomic(base + GICD_CTLR, val, !(val & bit), 350 1, USEC_PER_SEC); 351 if (ret == -ETIMEDOUT) 352 pr_err_ratelimited("RWP timeout, gone fishing\n"); 353 } 354 355 /* Wait for completion of a distributor change */ 356 static void gic_dist_wait_for_rwp(void) 357 { 358 gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP); 359 } 360 361 /* Wait for completion of a redistributor change */ 362 static void gic_redist_wait_for_rwp(void) 363 { 364 gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP); 365 } 366 367 static void gic_enable_redist(bool enable) 368 { 369 void __iomem *rbase; 370 u32 val; 371 int ret; 372 373 if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 374 return; 375 376 rbase = gic_data_rdist_rd_base(); 377 378 val = readl_relaxed(rbase + GICR_WAKER); 379 if (enable) 380 /* Wake up this CPU redistributor */ 381 val &= ~GICR_WAKER_ProcessorSleep; 382 else 383 val |= GICR_WAKER_ProcessorSleep; 384 writel_relaxed(val, rbase + GICR_WAKER); 385 386 if (!enable) { /* Check that GICR_WAKER is writeable */ 387 val = readl_relaxed(rbase + GICR_WAKER); 388 if (!(val & GICR_WAKER_ProcessorSleep)) 389 return; /* No PM support in this redistributor */ 390 } 391 392 ret = readl_relaxed_poll_timeout_atomic(rbase + GICR_WAKER, val, 393 enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep), 394 1, USEC_PER_SEC); 395 if (ret == -ETIMEDOUT) { 396 pr_err_ratelimited("redistributor failed to %s...\n", 397 enable ? "wakeup" : "sleep"); 398 } 399 } 400 401 /* 402 * Routines to disable, enable, EOI and route interrupts 403 */ 404 static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) 405 { 406 switch (get_intid_range(d)) { 407 case SGI_RANGE: 408 case PPI_RANGE: 409 case SPI_RANGE: 410 *index = d->hwirq; 411 return offset; 412 case EPPI_RANGE: 413 /* 414 * Contrary to the ESPI range, the EPPI range is contiguous 415 * to the PPI range in the registers, so let's adjust the 416 * displacement accordingly. Consistency is overrated. 417 */ 418 *index = d->hwirq - EPPI_BASE_INTID + 32; 419 return offset; 420 case ESPI_RANGE: 421 *index = d->hwirq - ESPI_BASE_INTID; 422 switch (offset) { 423 case GICD_ISENABLER: 424 return GICD_ISENABLERnE; 425 case GICD_ICENABLER: 426 return GICD_ICENABLERnE; 427 case GICD_ISPENDR: 428 return GICD_ISPENDRnE; 429 case GICD_ICPENDR: 430 return GICD_ICPENDRnE; 431 case GICD_ISACTIVER: 432 return GICD_ISACTIVERnE; 433 case GICD_ICACTIVER: 434 return GICD_ICACTIVERnE; 435 case GICD_IPRIORITYR: 436 return GICD_IPRIORITYRnE; 437 case GICD_ICFGR: 438 return GICD_ICFGRnE; 439 case GICD_IROUTER: 440 return GICD_IROUTERnE; 441 default: 442 break; 443 } 444 break; 445 default: 446 break; 447 } 448 449 WARN_ON(1); 450 *index = d->hwirq; 451 return offset; 452 } 453 454 static int gic_peek_irq(struct irq_data *d, u32 offset) 455 { 456 void __iomem *base; 457 u32 index, mask; 458 459 offset = convert_offset_index(d, offset, &index); 460 mask = 1 << (index % 32); 461 462 if (gic_irq_in_rdist(d)) 463 base = gic_data_rdist_sgi_base(); 464 else 465 base = gic_dist_base_alias(d); 466 467 return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask); 468 } 469 470 static void gic_poke_irq(struct irq_data *d, u32 offset) 471 { 472 void __iomem *base; 473 u32 index, mask; 474 475 offset = convert_offset_index(d, offset, &index); 476 mask = 1 << (index % 32); 477 478 if (gic_irq_in_rdist(d)) 479 base = gic_data_rdist_sgi_base(); 480 else 481 base = gic_data.dist_base; 482 483 writel_relaxed(mask, base + offset + (index / 32) * 4); 484 } 485 486 static void gic_mask_irq(struct irq_data *d) 487 { 488 gic_poke_irq(d, GICD_ICENABLER); 489 if (gic_irq_in_rdist(d)) 490 gic_redist_wait_for_rwp(); 491 else 492 gic_dist_wait_for_rwp(); 493 } 494 495 static void gic_eoimode1_mask_irq(struct irq_data *d) 496 { 497 gic_mask_irq(d); 498 /* 499 * When masking a forwarded interrupt, make sure it is 500 * deactivated as well. 501 * 502 * This ensures that an interrupt that is getting 503 * disabled/masked will not get "stuck", because there is 504 * noone to deactivate it (guest is being terminated). 505 */ 506 if (irqd_is_forwarded_to_vcpu(d)) 507 gic_poke_irq(d, GICD_ICACTIVER); 508 } 509 510 static void gic_unmask_irq(struct irq_data *d) 511 { 512 gic_poke_irq(d, GICD_ISENABLER); 513 } 514 515 static inline bool gic_supports_nmi(void) 516 { 517 return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 518 static_branch_likely(&supports_pseudo_nmis); 519 } 520 521 static int gic_irq_set_irqchip_state(struct irq_data *d, 522 enum irqchip_irq_state which, bool val) 523 { 524 u32 reg; 525 526 if (d->hwirq >= 8192) /* SGI/PPI/SPI only */ 527 return -EINVAL; 528 529 switch (which) { 530 case IRQCHIP_STATE_PENDING: 531 reg = val ? GICD_ISPENDR : GICD_ICPENDR; 532 break; 533 534 case IRQCHIP_STATE_ACTIVE: 535 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 536 break; 537 538 case IRQCHIP_STATE_MASKED: 539 if (val) { 540 gic_mask_irq(d); 541 return 0; 542 } 543 reg = GICD_ISENABLER; 544 break; 545 546 default: 547 return -EINVAL; 548 } 549 550 gic_poke_irq(d, reg); 551 552 /* 553 * Force read-back to guarantee that the active state has taken 554 * effect, and won't race with a guest-driven deactivation. 555 */ 556 if (reg == GICD_ISACTIVER) 557 gic_peek_irq(d, reg); 558 return 0; 559 } 560 561 static int gic_irq_get_irqchip_state(struct irq_data *d, 562 enum irqchip_irq_state which, bool *val) 563 { 564 if (d->hwirq >= 8192) /* PPI/SPI only */ 565 return -EINVAL; 566 567 switch (which) { 568 case IRQCHIP_STATE_PENDING: 569 *val = gic_peek_irq(d, GICD_ISPENDR); 570 break; 571 572 case IRQCHIP_STATE_ACTIVE: 573 *val = gic_peek_irq(d, GICD_ISACTIVER); 574 break; 575 576 case IRQCHIP_STATE_MASKED: 577 *val = !gic_peek_irq(d, GICD_ISENABLER); 578 break; 579 580 default: 581 return -EINVAL; 582 } 583 584 return 0; 585 } 586 587 static void gic_irq_set_prio(struct irq_data *d, u8 prio) 588 { 589 void __iomem *base = gic_dist_base(d); 590 u32 offset, index; 591 592 offset = convert_offset_index(d, GICD_IPRIORITYR, &index); 593 594 writeb_relaxed(prio, base + offset + index); 595 } 596 597 static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) 598 { 599 switch (__get_intid_range(hwirq)) { 600 case PPI_RANGE: 601 return hwirq - 16; 602 case EPPI_RANGE: 603 return hwirq - EPPI_BASE_INTID + 16; 604 default: 605 unreachable(); 606 } 607 } 608 609 static u32 __gic_get_rdist_index(irq_hw_number_t hwirq) 610 { 611 switch (__get_intid_range(hwirq)) { 612 case SGI_RANGE: 613 case PPI_RANGE: 614 return hwirq; 615 case EPPI_RANGE: 616 return hwirq - EPPI_BASE_INTID + 32; 617 default: 618 unreachable(); 619 } 620 } 621 622 static u32 gic_get_rdist_index(struct irq_data *d) 623 { 624 return __gic_get_rdist_index(d->hwirq); 625 } 626 627 static int gic_irq_nmi_setup(struct irq_data *d) 628 { 629 struct irq_desc *desc = irq_to_desc(d->irq); 630 631 if (!gic_supports_nmi()) 632 return -EINVAL; 633 634 if (gic_peek_irq(d, GICD_ISENABLER)) { 635 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 636 return -EINVAL; 637 } 638 639 /* 640 * A secondary irq_chip should be in charge of LPI request, 641 * it should not be possible to get there 642 */ 643 if (WARN_ON(irqd_to_hwirq(d) >= 8192)) 644 return -EINVAL; 645 646 /* desc lock should already be held */ 647 if (gic_irq_in_rdist(d)) { 648 u32 idx = gic_get_rdist_index(d); 649 650 /* 651 * Setting up a percpu interrupt as NMI, only switch handler 652 * for first NMI 653 */ 654 if (!refcount_inc_not_zero(&rdist_nmi_refs[idx])) { 655 refcount_set(&rdist_nmi_refs[idx], 1); 656 desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 657 } 658 } else { 659 desc->handle_irq = handle_fasteoi_nmi; 660 } 661 662 gic_irq_set_prio(d, dist_prio_nmi); 663 664 return 0; 665 } 666 667 static void gic_irq_nmi_teardown(struct irq_data *d) 668 { 669 struct irq_desc *desc = irq_to_desc(d->irq); 670 671 if (WARN_ON(!gic_supports_nmi())) 672 return; 673 674 if (gic_peek_irq(d, GICD_ISENABLER)) { 675 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 676 return; 677 } 678 679 /* 680 * A secondary irq_chip should be in charge of LPI request, 681 * it should not be possible to get there 682 */ 683 if (WARN_ON(irqd_to_hwirq(d) >= 8192)) 684 return; 685 686 /* desc lock should already be held */ 687 if (gic_irq_in_rdist(d)) { 688 u32 idx = gic_get_rdist_index(d); 689 690 /* Tearing down NMI, only switch handler for last NMI */ 691 if (refcount_dec_and_test(&rdist_nmi_refs[idx])) 692 desc->handle_irq = handle_percpu_devid_irq; 693 } else { 694 desc->handle_irq = handle_fasteoi_irq; 695 } 696 697 gic_irq_set_prio(d, dist_prio_irq); 698 } 699 700 static bool gic_arm64_erratum_2941627_needed(struct irq_data *d) 701 { 702 enum gic_intid_range range; 703 704 if (!static_branch_unlikely(&gic_arm64_2941627_erratum)) 705 return false; 706 707 range = get_intid_range(d); 708 709 /* 710 * The workaround is needed if the IRQ is an SPI and 711 * the target cpu is different from the one we are 712 * executing on. 713 */ 714 return (range == SPI_RANGE || range == ESPI_RANGE) && 715 !cpumask_test_cpu(raw_smp_processor_id(), 716 irq_data_get_effective_affinity_mask(d)); 717 } 718 719 static void gic_eoi_irq(struct irq_data *d) 720 { 721 write_gicreg(irqd_to_hwirq(d), ICC_EOIR1_EL1); 722 isb(); 723 724 if (gic_arm64_erratum_2941627_needed(d)) { 725 /* 726 * Make sure the GIC stream deactivate packet 727 * issued by ICC_EOIR1_EL1 has completed before 728 * deactivating through GICD_IACTIVER. 729 */ 730 dsb(sy); 731 gic_poke_irq(d, GICD_ICACTIVER); 732 } 733 } 734 735 static void gic_eoimode1_eoi_irq(struct irq_data *d) 736 { 737 /* 738 * No need to deactivate an LPI, or an interrupt that 739 * is is getting forwarded to a vcpu. 740 */ 741 if (irqd_to_hwirq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 742 return; 743 744 if (!gic_arm64_erratum_2941627_needed(d)) 745 gic_write_dir(irqd_to_hwirq(d)); 746 else 747 gic_poke_irq(d, GICD_ICACTIVER); 748 } 749 750 static int gic_set_type(struct irq_data *d, unsigned int type) 751 { 752 irq_hw_number_t irq = irqd_to_hwirq(d); 753 enum gic_intid_range range; 754 void __iomem *base; 755 u32 offset, index; 756 int ret; 757 758 range = get_intid_range(d); 759 760 /* Interrupt configuration for SGIs can't be changed */ 761 if (range == SGI_RANGE) 762 return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; 763 764 /* SPIs have restrictions on the supported types */ 765 if ((range == SPI_RANGE || range == ESPI_RANGE) && 766 type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) 767 return -EINVAL; 768 769 if (gic_irq_in_rdist(d)) 770 base = gic_data_rdist_sgi_base(); 771 else 772 base = gic_dist_base_alias(d); 773 774 offset = convert_offset_index(d, GICD_ICFGR, &index); 775 776 ret = gic_configure_irq(index, type, base + offset); 777 if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { 778 /* Misconfigured PPIs are usually not fatal */ 779 pr_warn("GIC: PPI INTID%ld is secure or misconfigured\n", irq); 780 ret = 0; 781 } 782 783 return ret; 784 } 785 786 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 787 { 788 if (get_intid_range(d) == SGI_RANGE) 789 return -EINVAL; 790 791 if (vcpu) 792 irqd_set_forwarded_to_vcpu(d); 793 else 794 irqd_clr_forwarded_to_vcpu(d); 795 return 0; 796 } 797 798 static u64 gic_cpu_to_affinity(int cpu) 799 { 800 u64 mpidr = cpu_logical_map(cpu); 801 u64 aff; 802 803 /* ASR8601 needs to have its affinities shifted down... */ 804 if (unlikely(gic_data.flags & FLAGS_WORKAROUND_ASR_ERRATUM_8601001)) 805 mpidr = (MPIDR_AFFINITY_LEVEL(mpidr, 1) | 806 (MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8)); 807 808 aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 809 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 810 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 811 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 812 813 return aff; 814 } 815 816 static void gic_deactivate_unhandled(u32 irqnr) 817 { 818 if (static_branch_likely(&supports_deactivate_key)) { 819 if (irqnr < 8192) 820 gic_write_dir(irqnr); 821 } else { 822 write_gicreg(irqnr, ICC_EOIR1_EL1); 823 isb(); 824 } 825 } 826 827 /* 828 * Follow a read of the IAR with any HW maintenance that needs to happen prior 829 * to invoking the relevant IRQ handler. We must do two things: 830 * 831 * (1) Ensure instruction ordering between a read of IAR and subsequent 832 * instructions in the IRQ handler using an ISB. 833 * 834 * It is possible for the IAR to report an IRQ which was signalled *after* 835 * the CPU took an IRQ exception as multiple interrupts can race to be 836 * recognized by the GIC, earlier interrupts could be withdrawn, and/or 837 * later interrupts could be prioritized by the GIC. 838 * 839 * For devices which are tightly coupled to the CPU, such as PMUs, a 840 * context synchronization event is necessary to ensure that system 841 * register state is not stale, as these may have been indirectly written 842 * *after* exception entry. 843 * 844 * (2) Execute an interrupt priority drop when EOI mode 1 is in use. 845 */ 846 static inline void gic_complete_ack(u32 irqnr) 847 { 848 if (static_branch_likely(&supports_deactivate_key)) 849 write_gicreg(irqnr, ICC_EOIR1_EL1); 850 851 isb(); 852 } 853 854 static bool gic_rpr_is_nmi_prio(void) 855 { 856 if (!gic_supports_nmi()) 857 return false; 858 859 return unlikely(gic_read_rpr() == GICV3_PRIO_NMI); 860 } 861 862 static bool gic_irqnr_is_special(u32 irqnr) 863 { 864 return irqnr >= 1020 && irqnr <= 1023; 865 } 866 867 static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs) 868 { 869 if (gic_irqnr_is_special(irqnr)) 870 return; 871 872 gic_complete_ack(irqnr); 873 874 if (generic_handle_domain_irq(gic_data.domain, irqnr)) { 875 WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr); 876 gic_deactivate_unhandled(irqnr); 877 } 878 } 879 880 static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 881 { 882 if (gic_irqnr_is_special(irqnr)) 883 return; 884 885 gic_complete_ack(irqnr); 886 887 if (generic_handle_domain_nmi(gic_data.domain, irqnr)) { 888 WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr); 889 gic_deactivate_unhandled(irqnr); 890 } 891 } 892 893 /* 894 * An exception has been taken from a context with IRQs enabled, and this could 895 * be an IRQ or an NMI. 896 * 897 * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear 898 * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning, 899 * after handling any NMI but before handling any IRQ. 900 * 901 * The entry code has performed IRQ entry, and if an NMI is detected we must 902 * perform NMI entry/exit around invoking the handler. 903 */ 904 static void __gic_handle_irq_from_irqson(struct pt_regs *regs) 905 { 906 bool is_nmi; 907 u32 irqnr; 908 909 irqnr = gic_read_iar(); 910 911 is_nmi = gic_rpr_is_nmi_prio(); 912 913 if (is_nmi) { 914 nmi_enter(); 915 __gic_handle_nmi(irqnr, regs); 916 nmi_exit(); 917 } 918 919 if (gic_prio_masking_enabled()) { 920 gic_pmr_mask_irqs(); 921 gic_arch_enable_irqs(); 922 } 923 924 if (!is_nmi) 925 __gic_handle_irq(irqnr, regs); 926 } 927 928 /* 929 * An exception has been taken from a context with IRQs disabled, which can only 930 * be an NMI. 931 * 932 * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave 933 * DAIF.IF (and ICC_PMR_EL1) unchanged. 934 * 935 * The entry code has performed NMI entry. 936 */ 937 static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs) 938 { 939 u64 pmr; 940 u32 irqnr; 941 942 /* 943 * We were in a context with IRQs disabled. However, the 944 * entry code has set PMR to a value that allows any 945 * interrupt to be acknowledged, and not just NMIs. This can 946 * lead to surprising effects if the NMI has been retired in 947 * the meantime, and that there is an IRQ pending. The IRQ 948 * would then be taken in NMI context, something that nobody 949 * wants to debug twice. 950 * 951 * Until we sort this, drop PMR again to a level that will 952 * actually only allow NMIs before reading IAR, and then 953 * restore it to what it was. 954 */ 955 pmr = gic_read_pmr(); 956 gic_pmr_mask_irqs(); 957 isb(); 958 irqnr = gic_read_iar(); 959 gic_write_pmr(pmr); 960 961 __gic_handle_nmi(irqnr, regs); 962 } 963 964 static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 965 { 966 if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs))) 967 __gic_handle_irq_from_irqsoff(regs); 968 else 969 __gic_handle_irq_from_irqson(regs); 970 } 971 972 static void __init gic_dist_init(void) 973 { 974 unsigned int i; 975 u64 affinity; 976 void __iomem *base = gic_data.dist_base; 977 u32 val; 978 979 /* Disable the distributor */ 980 writel_relaxed(0, base + GICD_CTLR); 981 gic_dist_wait_for_rwp(); 982 983 /* 984 * Configure SPIs as non-secure Group-1. This will only matter 985 * if the GIC only has a single security state. This will not 986 * do the right thing if the kernel is running in secure mode, 987 * but that's not the intended use case anyway. 988 */ 989 for (i = 32; i < GIC_LINE_NR; i += 32) 990 writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 991 992 /* Extended SPI range, not handled by the GICv2/GICv3 common code */ 993 for (i = 0; i < GIC_ESPI_NR; i += 32) { 994 writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8); 995 writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8); 996 } 997 998 for (i = 0; i < GIC_ESPI_NR; i += 32) 999 writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8); 1000 1001 for (i = 0; i < GIC_ESPI_NR; i += 16) 1002 writel_relaxed(0, base + GICD_ICFGRnE + i / 4); 1003 1004 for (i = 0; i < GIC_ESPI_NR; i += 4) 1005 writel_relaxed(REPEAT_BYTE_U32(dist_prio_irq), 1006 base + GICD_IPRIORITYRnE + i); 1007 1008 /* Now do the common stuff */ 1009 gic_dist_config(base, GIC_LINE_NR, dist_prio_irq); 1010 1011 val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; 1012 if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { 1013 pr_info("Enabling SGIs without active state\n"); 1014 val |= GICD_CTLR_nASSGIreq; 1015 } 1016 1017 /* Enable distributor with ARE, Group1, and wait for it to drain */ 1018 writel_relaxed(val, base + GICD_CTLR); 1019 gic_dist_wait_for_rwp(); 1020 1021 /* 1022 * Set all global interrupts to the boot CPU only. ARE must be 1023 * enabled. 1024 */ 1025 affinity = gic_cpu_to_affinity(smp_processor_id()); 1026 for (i = 32; i < GIC_LINE_NR; i++) 1027 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 1028 1029 for (i = 0; i < GIC_ESPI_NR; i++) 1030 gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8); 1031 } 1032 1033 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 1034 { 1035 int ret = -ENODEV; 1036 int i; 1037 1038 for (i = 0; i < gic_data.nr_redist_regions; i++) { 1039 void __iomem *ptr = gic_data.redist_regions[i].redist_base; 1040 u64 typer; 1041 u32 reg; 1042 1043 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 1044 if (reg != GIC_PIDR2_ARCH_GICv3 && 1045 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 1046 pr_warn("No redistributor present @%p\n", ptr); 1047 break; 1048 } 1049 1050 do { 1051 typer = gic_read_typer(ptr + GICR_TYPER); 1052 ret = fn(gic_data.redist_regions + i, ptr); 1053 if (!ret) 1054 return 0; 1055 1056 if (gic_data.redist_regions[i].single_redist) 1057 break; 1058 1059 if (gic_data.redist_stride) { 1060 ptr += gic_data.redist_stride; 1061 } else { 1062 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 1063 if (typer & GICR_TYPER_VLPIS) 1064 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 1065 } 1066 } while (!(typer & GICR_TYPER_LAST)); 1067 } 1068 1069 return ret ? -ENODEV : 0; 1070 } 1071 1072 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 1073 { 1074 unsigned long mpidr; 1075 u64 typer; 1076 u32 aff; 1077 1078 /* 1079 * Convert affinity to a 32bit value that can be matched to 1080 * GICR_TYPER bits [63:32]. 1081 */ 1082 mpidr = gic_cpu_to_affinity(smp_processor_id()); 1083 1084 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 1085 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 1086 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 1087 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 1088 1089 typer = gic_read_typer(ptr + GICR_TYPER); 1090 if ((typer >> 32) == aff) { 1091 u64 offset = ptr - region->redist_base; 1092 raw_spin_lock_init(&gic_data_rdist()->rd_lock); 1093 gic_data_rdist_rd_base() = ptr; 1094 gic_data_rdist()->phys_base = region->phys_base + offset; 1095 1096 pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 1097 smp_processor_id(), mpidr, 1098 (int)(region - gic_data.redist_regions), 1099 &gic_data_rdist()->phys_base); 1100 return 0; 1101 } 1102 1103 /* Try next one */ 1104 return 1; 1105 } 1106 1107 static int gic_populate_rdist(void) 1108 { 1109 if (gic_iterate_rdists(__gic_populate_rdist) == 0) 1110 return 0; 1111 1112 /* We couldn't even deal with ourselves... */ 1113 WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 1114 smp_processor_id(), 1115 (unsigned long)cpu_logical_map(smp_processor_id())); 1116 return -ENODEV; 1117 } 1118 1119 static int __gic_update_rdist_properties(struct redist_region *region, 1120 void __iomem *ptr) 1121 { 1122 u64 typer = gic_read_typer(ptr + GICR_TYPER); 1123 u32 ctlr = readl_relaxed(ptr + GICR_CTLR); 1124 1125 /* Boot-time cleanup */ 1126 if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { 1127 u64 val; 1128 1129 /* Deactivate any present vPE */ 1130 val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER); 1131 if (val & GICR_VPENDBASER_Valid) 1132 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, 1133 ptr + SZ_128K + GICR_VPENDBASER); 1134 1135 /* Mark the VPE table as invalid */ 1136 val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER); 1137 val &= ~GICR_VPROPBASER_4_1_VALID; 1138 gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER); 1139 } 1140 1141 gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 1142 1143 /* 1144 * TYPER.RVPEID implies some form of DirectLPI, no matter what the 1145 * doc says... :-/ And CTLR.IR implies another subset of DirectLPI 1146 * that the ITS driver can make use of for LPIs (and not VLPIs). 1147 * 1148 * These are 3 different ways to express the same thing, depending 1149 * on the revision of the architecture and its relaxations over 1150 * time. Just group them under the 'direct_lpi' banner. 1151 */ 1152 gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID); 1153 gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) | 1154 !!(ctlr & GICR_CTLR_IR) | 1155 gic_data.rdists.has_rvpeid); 1156 gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY); 1157 1158 /* Detect non-sensical configurations */ 1159 if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) { 1160 gic_data.rdists.has_direct_lpi = false; 1161 gic_data.rdists.has_vlpis = false; 1162 gic_data.rdists.has_rvpeid = false; 1163 } 1164 1165 gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); 1166 1167 return 1; 1168 } 1169 1170 static void gic_update_rdist_properties(void) 1171 { 1172 gic_data.ppi_nr = UINT_MAX; 1173 gic_iterate_rdists(__gic_update_rdist_properties); 1174 if (WARN_ON(gic_data.ppi_nr == UINT_MAX)) 1175 gic_data.ppi_nr = 0; 1176 pr_info("GICv3 features: %d PPIs%s%s\n", 1177 gic_data.ppi_nr, 1178 gic_data.has_rss ? ", RSS" : "", 1179 gic_data.rdists.has_direct_lpi ? ", DirectLPI" : ""); 1180 1181 if (gic_data.rdists.has_vlpis) 1182 pr_info("GICv4 features: %s%s%s\n", 1183 gic_data.rdists.has_direct_lpi ? "DirectLPI " : "", 1184 gic_data.rdists.has_rvpeid ? "RVPEID " : "", 1185 gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); 1186 } 1187 1188 static void gic_cpu_sys_reg_enable(void) 1189 { 1190 /* 1191 * Need to check that the SRE bit has actually been set. If 1192 * not, it means that SRE is disabled at EL2. We're going to 1193 * die painfully, and there is nothing we can do about it. 1194 * 1195 * Kindly inform the luser. 1196 */ 1197 if (!gic_enable_sre()) 1198 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 1199 1200 } 1201 1202 static void gic_cpu_sys_reg_init(void) 1203 { 1204 int i, cpu = smp_processor_id(); 1205 u64 mpidr = gic_cpu_to_affinity(cpu); 1206 u64 need_rss = MPIDR_RS(mpidr); 1207 bool group0; 1208 u32 pribits; 1209 1210 pribits = gic_get_pribits(); 1211 1212 group0 = gic_has_group0(); 1213 1214 /* Set priority mask register */ 1215 if (!gic_prio_masking_enabled()) { 1216 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 1217 } else if (gic_supports_nmi()) { 1218 /* 1219 * Check that all CPUs use the same priority space. 1220 * 1221 * If there's a mismatch with the boot CPU, the system is 1222 * likely to die as interrupt masking will not work properly on 1223 * all CPUs. 1224 */ 1225 WARN_ON(group0 != cpus_have_group0); 1226 WARN_ON(gic_dist_security_disabled() != cpus_have_security_disabled); 1227 } 1228 1229 /* 1230 * Some firmwares hand over to the kernel with the BPR changed from 1231 * its reset value (and with a value large enough to prevent 1232 * any pre-emptive interrupts from working at all). Writing a zero 1233 * to BPR restores is reset value. 1234 */ 1235 gic_write_bpr1(0); 1236 1237 if (static_branch_likely(&supports_deactivate_key)) { 1238 /* EOI drops priority only (mode 1) */ 1239 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 1240 } else { 1241 /* EOI deactivates interrupt too (mode 0) */ 1242 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 1243 } 1244 1245 /* Always whack Group0 before Group1 */ 1246 if (group0) { 1247 switch(pribits) { 1248 case 8: 1249 case 7: 1250 write_gicreg(0, ICC_AP0R3_EL1); 1251 write_gicreg(0, ICC_AP0R2_EL1); 1252 fallthrough; 1253 case 6: 1254 write_gicreg(0, ICC_AP0R1_EL1); 1255 fallthrough; 1256 case 5: 1257 case 4: 1258 write_gicreg(0, ICC_AP0R0_EL1); 1259 } 1260 1261 isb(); 1262 } 1263 1264 switch(pribits) { 1265 case 8: 1266 case 7: 1267 write_gicreg(0, ICC_AP1R3_EL1); 1268 write_gicreg(0, ICC_AP1R2_EL1); 1269 fallthrough; 1270 case 6: 1271 write_gicreg(0, ICC_AP1R1_EL1); 1272 fallthrough; 1273 case 5: 1274 case 4: 1275 write_gicreg(0, ICC_AP1R0_EL1); 1276 } 1277 1278 isb(); 1279 1280 /* ... and let's hit the road... */ 1281 gic_write_grpen1(1); 1282 1283 /* Keep the RSS capability status in per_cpu variable */ 1284 per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 1285 1286 /* Check all the CPUs have capable of sending SGIs to other CPUs */ 1287 for_each_online_cpu(i) { 1288 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 1289 1290 need_rss |= MPIDR_RS(gic_cpu_to_affinity(i)); 1291 if (need_rss && (!have_rss)) 1292 pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 1293 cpu, (unsigned long)mpidr, 1294 i, (unsigned long)gic_cpu_to_affinity(i)); 1295 } 1296 1297 /** 1298 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 1299 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 1300 * UNPREDICTABLE choice of : 1301 * - The write is ignored. 1302 * - The RS field is treated as 0. 1303 */ 1304 if (need_rss && (!gic_data.has_rss)) 1305 pr_crit_once("RSS is required but GICD doesn't support it\n"); 1306 } 1307 1308 static bool gicv3_nolpi; 1309 1310 static int __init gicv3_nolpi_cfg(char *buf) 1311 { 1312 return kstrtobool(buf, &gicv3_nolpi); 1313 } 1314 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 1315 1316 static int gic_dist_supports_lpis(void) 1317 { 1318 return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 1319 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 1320 !gicv3_nolpi); 1321 } 1322 1323 static void gic_cpu_init(void) 1324 { 1325 void __iomem *rbase; 1326 int i; 1327 1328 /* Register ourselves with the rest of the world */ 1329 if (gic_populate_rdist()) 1330 return; 1331 1332 gic_enable_redist(true); 1333 1334 WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) && 1335 !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange), 1336 "Distributor has extended ranges, but CPU%d doesn't\n", 1337 smp_processor_id()); 1338 1339 rbase = gic_data_rdist_sgi_base(); 1340 1341 /* Configure SGIs/PPIs as non-secure Group-1 */ 1342 for (i = 0; i < gic_data.ppi_nr + SGI_NR; i += 32) 1343 writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); 1344 1345 gic_cpu_config(rbase, gic_data.ppi_nr + SGI_NR, dist_prio_irq); 1346 gic_redist_wait_for_rwp(); 1347 1348 /* initialise system registers */ 1349 gic_cpu_sys_reg_init(); 1350 } 1351 1352 #ifdef CONFIG_SMP 1353 1354 #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 1355 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 1356 1357 /* 1358 * gic_starting_cpu() is called after the last point where cpuhp is allowed 1359 * to fail. So pre check for problems earlier. 1360 */ 1361 static int gic_check_rdist(unsigned int cpu) 1362 { 1363 if (cpumask_test_cpu(cpu, &broken_rdists)) 1364 return -EINVAL; 1365 1366 return 0; 1367 } 1368 1369 static int gic_starting_cpu(unsigned int cpu) 1370 { 1371 gic_cpu_sys_reg_enable(); 1372 gic_cpu_init(); 1373 1374 if (gic_dist_supports_lpis()) 1375 its_cpu_init(); 1376 1377 return 0; 1378 } 1379 1380 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 1381 unsigned long cluster_id) 1382 { 1383 int next_cpu, cpu = *base_cpu; 1384 unsigned long mpidr; 1385 u16 tlist = 0; 1386 1387 mpidr = gic_cpu_to_affinity(cpu); 1388 1389 while (cpu < nr_cpu_ids) { 1390 tlist |= 1 << (mpidr & 0xf); 1391 1392 next_cpu = cpumask_next(cpu, mask); 1393 if (next_cpu >= nr_cpu_ids) 1394 goto out; 1395 cpu = next_cpu; 1396 1397 mpidr = gic_cpu_to_affinity(cpu); 1398 1399 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 1400 cpu--; 1401 goto out; 1402 } 1403 } 1404 out: 1405 *base_cpu = cpu; 1406 return tlist; 1407 } 1408 1409 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 1410 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 1411 << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 1412 1413 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 1414 { 1415 u64 val; 1416 1417 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 1418 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 1419 irq << ICC_SGI1R_SGI_ID_SHIFT | 1420 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 1421 MPIDR_TO_SGI_RS(cluster_id) | 1422 tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 1423 1424 pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 1425 gic_write_sgi1r(val); 1426 } 1427 1428 static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) 1429 { 1430 int cpu; 1431 1432 if (WARN_ON(d->hwirq >= 16)) 1433 return; 1434 1435 /* 1436 * Ensure that stores to Normal memory are visible to the 1437 * other CPUs before issuing the IPI. 1438 */ 1439 dsb(ishst); 1440 1441 for_each_cpu(cpu, mask) { 1442 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(gic_cpu_to_affinity(cpu)); 1443 u16 tlist; 1444 1445 tlist = gic_compute_target_list(&cpu, mask, cluster_id); 1446 gic_send_sgi(cluster_id, tlist, d->hwirq); 1447 } 1448 1449 /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 1450 isb(); 1451 } 1452 1453 static void __init gic_smp_init(void) 1454 { 1455 struct irq_fwspec sgi_fwspec = { 1456 .fwnode = gic_data.fwnode, 1457 .param_count = 1, 1458 }; 1459 int base_sgi; 1460 1461 cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN, 1462 "irqchip/arm/gicv3:checkrdist", 1463 gic_check_rdist, NULL); 1464 1465 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 1466 "irqchip/arm/gicv3:starting", 1467 gic_starting_cpu, NULL); 1468 1469 /* Register all 8 non-secure SGIs */ 1470 base_sgi = irq_domain_alloc_irqs(gic_data.domain, 8, NUMA_NO_NODE, &sgi_fwspec); 1471 if (WARN_ON(base_sgi <= 0)) 1472 return; 1473 1474 set_smp_ipi_range(base_sgi, 8); 1475 } 1476 1477 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1478 bool force) 1479 { 1480 unsigned int cpu; 1481 u32 offset, index; 1482 void __iomem *reg; 1483 int enabled; 1484 u64 val; 1485 1486 if (force) 1487 cpu = cpumask_first(mask_val); 1488 else 1489 cpu = cpumask_any_and(mask_val, cpu_online_mask); 1490 1491 if (cpu >= nr_cpu_ids) 1492 return -EINVAL; 1493 1494 if (gic_irq_in_rdist(d)) 1495 return -EINVAL; 1496 1497 /* If interrupt was enabled, disable it first */ 1498 enabled = gic_peek_irq(d, GICD_ISENABLER); 1499 if (enabled) 1500 gic_mask_irq(d); 1501 1502 offset = convert_offset_index(d, GICD_IROUTER, &index); 1503 reg = gic_dist_base(d) + offset + (index * 8); 1504 val = gic_cpu_to_affinity(cpu); 1505 1506 gic_write_irouter(val, reg); 1507 1508 /* 1509 * If the interrupt was enabled, enabled it again. Otherwise, 1510 * just wait for the distributor to have digested our changes. 1511 */ 1512 if (enabled) 1513 gic_unmask_irq(d); 1514 1515 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1516 1517 return IRQ_SET_MASK_OK_DONE; 1518 } 1519 #else 1520 #define gic_set_affinity NULL 1521 #define gic_ipi_send_mask NULL 1522 #define gic_smp_init() do { } while(0) 1523 #endif 1524 1525 static int gic_retrigger(struct irq_data *data) 1526 { 1527 return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true); 1528 } 1529 1530 #ifdef CONFIG_CPU_PM 1531 static int gic_cpu_pm_notifier(struct notifier_block *self, 1532 unsigned long cmd, void *v) 1533 { 1534 if (cmd == CPU_PM_EXIT || cmd == CPU_PM_ENTER_FAILED) { 1535 if (gic_dist_security_disabled()) 1536 gic_enable_redist(true); 1537 gic_cpu_sys_reg_enable(); 1538 gic_cpu_sys_reg_init(); 1539 } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 1540 gic_write_grpen1(0); 1541 gic_enable_redist(false); 1542 } 1543 return NOTIFY_OK; 1544 } 1545 1546 static struct notifier_block gic_cpu_pm_notifier_block = { 1547 .notifier_call = gic_cpu_pm_notifier, 1548 }; 1549 1550 static void gic_cpu_pm_init(void) 1551 { 1552 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 1553 } 1554 1555 #else 1556 static inline void gic_cpu_pm_init(void) { } 1557 #endif /* CONFIG_CPU_PM */ 1558 1559 static struct irq_chip gic_chip = { 1560 .name = "GICv3", 1561 .irq_mask = gic_mask_irq, 1562 .irq_unmask = gic_unmask_irq, 1563 .irq_eoi = gic_eoi_irq, 1564 .irq_set_type = gic_set_type, 1565 .irq_set_affinity = gic_set_affinity, 1566 .irq_retrigger = gic_retrigger, 1567 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1568 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1569 .irq_nmi_setup = gic_irq_nmi_setup, 1570 .irq_nmi_teardown = gic_irq_nmi_teardown, 1571 .ipi_send_mask = gic_ipi_send_mask, 1572 .flags = IRQCHIP_SET_TYPE_MASKED | 1573 IRQCHIP_SKIP_SET_WAKE | 1574 IRQCHIP_MASK_ON_SUSPEND, 1575 }; 1576 1577 static struct irq_chip gic_eoimode1_chip = { 1578 .name = "GICv3", 1579 .irq_mask = gic_eoimode1_mask_irq, 1580 .irq_unmask = gic_unmask_irq, 1581 .irq_eoi = gic_eoimode1_eoi_irq, 1582 .irq_set_type = gic_set_type, 1583 .irq_set_affinity = gic_set_affinity, 1584 .irq_retrigger = gic_retrigger, 1585 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1586 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1587 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1588 .irq_nmi_setup = gic_irq_nmi_setup, 1589 .irq_nmi_teardown = gic_irq_nmi_teardown, 1590 .ipi_send_mask = gic_ipi_send_mask, 1591 .flags = IRQCHIP_SET_TYPE_MASKED | 1592 IRQCHIP_SKIP_SET_WAKE | 1593 IRQCHIP_MASK_ON_SUSPEND, 1594 }; 1595 1596 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1597 irq_hw_number_t hw) 1598 { 1599 struct irq_chip *chip = &gic_chip; 1600 struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq)); 1601 1602 if (static_branch_likely(&supports_deactivate_key)) 1603 chip = &gic_eoimode1_chip; 1604 1605 switch (__get_intid_range(hw)) { 1606 case SGI_RANGE: 1607 case PPI_RANGE: 1608 case EPPI_RANGE: 1609 irq_set_percpu_devid(irq); 1610 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1611 handle_percpu_devid_irq, NULL, NULL); 1612 break; 1613 1614 case SPI_RANGE: 1615 case ESPI_RANGE: 1616 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1617 handle_fasteoi_irq, NULL, NULL); 1618 irq_set_probe(irq); 1619 irqd_set_single_target(irqd); 1620 break; 1621 1622 case LPI_RANGE: 1623 if (!gic_dist_supports_lpis()) 1624 return -EPERM; 1625 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1626 handle_fasteoi_irq, NULL, NULL); 1627 break; 1628 1629 default: 1630 return -EPERM; 1631 } 1632 1633 /* Prevents SW retriggers which mess up the ACK/EOI ordering */ 1634 irqd_set_handle_enforce_irqctx(irqd); 1635 return 0; 1636 } 1637 1638 static int gic_irq_domain_translate(struct irq_domain *d, 1639 struct irq_fwspec *fwspec, 1640 unsigned long *hwirq, 1641 unsigned int *type) 1642 { 1643 if (fwspec->param_count == 1 && fwspec->param[0] < 16) { 1644 *hwirq = fwspec->param[0]; 1645 *type = IRQ_TYPE_EDGE_RISING; 1646 return 0; 1647 } 1648 1649 if (is_of_node(fwspec->fwnode)) { 1650 if (fwspec->param_count < 3) 1651 return -EINVAL; 1652 1653 switch (fwspec->param[0]) { 1654 case 0: /* SPI */ 1655 *hwirq = fwspec->param[1] + 32; 1656 break; 1657 case 1: /* PPI */ 1658 *hwirq = fwspec->param[1] + 16; 1659 break; 1660 case 2: /* ESPI */ 1661 *hwirq = fwspec->param[1] + ESPI_BASE_INTID; 1662 break; 1663 case 3: /* EPPI */ 1664 *hwirq = fwspec->param[1] + EPPI_BASE_INTID; 1665 break; 1666 case GIC_IRQ_TYPE_LPI: /* LPI */ 1667 *hwirq = fwspec->param[1]; 1668 break; 1669 case GIC_IRQ_TYPE_PARTITION: 1670 *hwirq = fwspec->param[1]; 1671 if (fwspec->param[1] >= 16) 1672 *hwirq += EPPI_BASE_INTID - 16; 1673 else 1674 *hwirq += 16; 1675 break; 1676 default: 1677 return -EINVAL; 1678 } 1679 1680 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1681 1682 /* 1683 * Make it clear that broken DTs are... broken. 1684 * Partitioned PPIs are an unfortunate exception. 1685 */ 1686 WARN_ON(*type == IRQ_TYPE_NONE && 1687 fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1688 return 0; 1689 } 1690 1691 if (is_fwnode_irqchip(fwspec->fwnode)) { 1692 if(fwspec->param_count != 2) 1693 return -EINVAL; 1694 1695 if (fwspec->param[0] < 16) { 1696 pr_err(FW_BUG "Illegal GSI%d translation request\n", 1697 fwspec->param[0]); 1698 return -EINVAL; 1699 } 1700 1701 *hwirq = fwspec->param[0]; 1702 *type = fwspec->param[1]; 1703 1704 WARN_ON(*type == IRQ_TYPE_NONE); 1705 return 0; 1706 } 1707 1708 return -EINVAL; 1709 } 1710 1711 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1712 unsigned int nr_irqs, void *arg) 1713 { 1714 int i, ret; 1715 irq_hw_number_t hwirq; 1716 unsigned int type = IRQ_TYPE_NONE; 1717 struct irq_fwspec *fwspec = arg; 1718 1719 ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1720 if (ret) 1721 return ret; 1722 1723 for (i = 0; i < nr_irqs; i++) { 1724 ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 1725 if (ret) 1726 return ret; 1727 } 1728 1729 return 0; 1730 } 1731 1732 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1733 unsigned int nr_irqs) 1734 { 1735 int i; 1736 1737 for (i = 0; i < nr_irqs; i++) { 1738 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1739 irq_set_handler(virq + i, NULL); 1740 irq_domain_reset_irq_data(d); 1741 } 1742 } 1743 1744 static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec, 1745 irq_hw_number_t hwirq) 1746 { 1747 enum gic_intid_range range; 1748 1749 if (!gic_data.ppi_descs) 1750 return false; 1751 1752 if (!is_of_node(fwspec->fwnode)) 1753 return false; 1754 1755 if (fwspec->param_count < 4 || !fwspec->param[3]) 1756 return false; 1757 1758 range = __get_intid_range(hwirq); 1759 if (range != PPI_RANGE && range != EPPI_RANGE) 1760 return false; 1761 1762 return true; 1763 } 1764 1765 static int gic_irq_domain_select(struct irq_domain *d, 1766 struct irq_fwspec *fwspec, 1767 enum irq_domain_bus_token bus_token) 1768 { 1769 unsigned int type, ppi_idx; 1770 irq_hw_number_t hwirq; 1771 int ret; 1772 1773 /* Not for us */ 1774 if (fwspec->fwnode != d->fwnode) 1775 return 0; 1776 1777 /* Handle pure domain searches */ 1778 if (!fwspec->param_count) 1779 return d->bus_token == bus_token; 1780 1781 /* If this is not DT, then we have a single domain */ 1782 if (!is_of_node(fwspec->fwnode)) 1783 return 1; 1784 1785 ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type); 1786 if (WARN_ON_ONCE(ret)) 1787 return 0; 1788 1789 if (!fwspec_is_partitioned_ppi(fwspec, hwirq)) 1790 return d == gic_data.domain; 1791 1792 /* 1793 * If this is a PPI and we have a 4th (non-null) parameter, 1794 * then we need to match the partition domain. 1795 */ 1796 ppi_idx = __gic_get_ppi_index(hwirq); 1797 return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]); 1798 } 1799 1800 static const struct irq_domain_ops gic_irq_domain_ops = { 1801 .translate = gic_irq_domain_translate, 1802 .alloc = gic_irq_domain_alloc, 1803 .free = gic_irq_domain_free, 1804 .select = gic_irq_domain_select, 1805 }; 1806 1807 static int partition_domain_translate(struct irq_domain *d, 1808 struct irq_fwspec *fwspec, 1809 unsigned long *hwirq, 1810 unsigned int *type) 1811 { 1812 unsigned long ppi_intid; 1813 struct device_node *np; 1814 unsigned int ppi_idx; 1815 int ret; 1816 1817 if (!gic_data.ppi_descs) 1818 return -ENOMEM; 1819 1820 np = of_find_node_by_phandle(fwspec->param[3]); 1821 if (WARN_ON(!np)) 1822 return -EINVAL; 1823 1824 ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type); 1825 if (WARN_ON_ONCE(ret)) 1826 return 0; 1827 1828 ppi_idx = __gic_get_ppi_index(ppi_intid); 1829 ret = partition_translate_id(gic_data.ppi_descs[ppi_idx], 1830 of_fwnode_handle(np)); 1831 if (ret < 0) 1832 return ret; 1833 1834 *hwirq = ret; 1835 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1836 1837 return 0; 1838 } 1839 1840 static const struct irq_domain_ops partition_domain_ops = { 1841 .translate = partition_domain_translate, 1842 .select = gic_irq_domain_select, 1843 }; 1844 1845 static bool gic_enable_quirk_msm8996(void *data) 1846 { 1847 struct gic_chip_data *d = data; 1848 1849 d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 1850 1851 return true; 1852 } 1853 1854 static bool gic_enable_quirk_cavium_38539(void *data) 1855 { 1856 struct gic_chip_data *d = data; 1857 1858 d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539; 1859 1860 return true; 1861 } 1862 1863 static bool gic_enable_quirk_hip06_07(void *data) 1864 { 1865 struct gic_chip_data *d = data; 1866 1867 /* 1868 * HIP06 GICD_IIDR clashes with GIC-600 product number (despite 1869 * not being an actual ARM implementation). The saving grace is 1870 * that GIC-600 doesn't have ESPI, so nothing to do in that case. 1871 * HIP07 doesn't even have a proper IIDR, and still pretends to 1872 * have ESPI. In both cases, put them right. 1873 */ 1874 if (d->rdists.gicd_typer & GICD_TYPER_ESPI) { 1875 /* Zero both ESPI and the RES0 field next to it... */ 1876 d->rdists.gicd_typer &= ~GENMASK(9, 8); 1877 return true; 1878 } 1879 1880 return false; 1881 } 1882 1883 #define T241_CHIPN_MASK GENMASK_ULL(45, 44) 1884 #define T241_CHIP_GICDA_OFFSET 0x1580000 1885 #define SMCCC_SOC_ID_T241 0x036b0241 1886 1887 static bool gic_enable_quirk_nvidia_t241(void *data) 1888 { 1889 s32 soc_id = arm_smccc_get_soc_id_version(); 1890 unsigned long chip_bmask = 0; 1891 phys_addr_t phys; 1892 u32 i; 1893 1894 /* Check JEP106 code for NVIDIA T241 chip (036b:0241) */ 1895 if ((soc_id < 0) || (soc_id != SMCCC_SOC_ID_T241)) 1896 return false; 1897 1898 /* Find the chips based on GICR regions PHYS addr */ 1899 for (i = 0; i < gic_data.nr_redist_regions; i++) { 1900 chip_bmask |= BIT(FIELD_GET(T241_CHIPN_MASK, 1901 (u64)gic_data.redist_regions[i].phys_base)); 1902 } 1903 1904 if (hweight32(chip_bmask) < 3) 1905 return false; 1906 1907 /* Setup GICD alias regions */ 1908 for (i = 0; i < ARRAY_SIZE(t241_dist_base_alias); i++) { 1909 if (chip_bmask & BIT(i)) { 1910 phys = gic_data.dist_phys_base + T241_CHIP_GICDA_OFFSET; 1911 phys |= FIELD_PREP(T241_CHIPN_MASK, i); 1912 t241_dist_base_alias[i] = ioremap(phys, SZ_64K); 1913 WARN_ON_ONCE(!t241_dist_base_alias[i]); 1914 } 1915 } 1916 static_branch_enable(&gic_nvidia_t241_erratum); 1917 return true; 1918 } 1919 1920 static bool gic_enable_quirk_asr8601(void *data) 1921 { 1922 struct gic_chip_data *d = data; 1923 1924 d->flags |= FLAGS_WORKAROUND_ASR_ERRATUM_8601001; 1925 1926 return true; 1927 } 1928 1929 static bool gic_enable_quirk_arm64_2941627(void *data) 1930 { 1931 static_branch_enable(&gic_arm64_2941627_erratum); 1932 return true; 1933 } 1934 1935 static bool gic_enable_quirk_rk3399(void *data) 1936 { 1937 struct gic_chip_data *d = data; 1938 1939 if (of_machine_is_compatible("rockchip,rk3399")) { 1940 d->flags |= FLAGS_WORKAROUND_INSECURE; 1941 return true; 1942 } 1943 1944 return false; 1945 } 1946 1947 static bool rd_set_non_coherent(void *data) 1948 { 1949 struct gic_chip_data *d = data; 1950 1951 d->rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; 1952 return true; 1953 } 1954 1955 static const struct gic_quirk gic_quirks[] = { 1956 { 1957 .desc = "GICv3: Qualcomm MSM8996 broken firmware", 1958 .compatible = "qcom,msm8996-gic-v3", 1959 .init = gic_enable_quirk_msm8996, 1960 }, 1961 { 1962 .desc = "GICv3: ASR erratum 8601001", 1963 .compatible = "asr,asr8601-gic-v3", 1964 .init = gic_enable_quirk_asr8601, 1965 }, 1966 { 1967 .desc = "GICv3: HIP06 erratum 161010803", 1968 .iidr = 0x0204043b, 1969 .mask = 0xffffffff, 1970 .init = gic_enable_quirk_hip06_07, 1971 }, 1972 { 1973 .desc = "GICv3: HIP07 erratum 161010803", 1974 .iidr = 0x00000000, 1975 .mask = 0xffffffff, 1976 .init = gic_enable_quirk_hip06_07, 1977 }, 1978 { 1979 /* 1980 * Reserved register accesses generate a Synchronous 1981 * External Abort. This erratum applies to: 1982 * - ThunderX: CN88xx 1983 * - OCTEON TX: CN83xx, CN81xx 1984 * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx* 1985 */ 1986 .desc = "GICv3: Cavium erratum 38539", 1987 .iidr = 0xa000034c, 1988 .mask = 0xe8f00fff, 1989 .init = gic_enable_quirk_cavium_38539, 1990 }, 1991 { 1992 .desc = "GICv3: NVIDIA erratum T241-FABRIC-4", 1993 .iidr = 0x0402043b, 1994 .mask = 0xffffffff, 1995 .init = gic_enable_quirk_nvidia_t241, 1996 }, 1997 { 1998 /* 1999 * GIC-700: 2941627 workaround - IP variant [0,1] 2000 * 2001 */ 2002 .desc = "GICv3: ARM64 erratum 2941627", 2003 .iidr = 0x0400043b, 2004 .mask = 0xff0e0fff, 2005 .init = gic_enable_quirk_arm64_2941627, 2006 }, 2007 { 2008 /* 2009 * GIC-700: 2941627 workaround - IP variant [2] 2010 */ 2011 .desc = "GICv3: ARM64 erratum 2941627", 2012 .iidr = 0x0402043b, 2013 .mask = 0xff0f0fff, 2014 .init = gic_enable_quirk_arm64_2941627, 2015 }, 2016 { 2017 .desc = "GICv3: non-coherent attribute", 2018 .property = "dma-noncoherent", 2019 .init = rd_set_non_coherent, 2020 }, 2021 { 2022 .desc = "GICv3: Insecure RK3399 integration", 2023 .iidr = 0x0000043b, 2024 .mask = 0xff000fff, 2025 .init = gic_enable_quirk_rk3399, 2026 }, 2027 { 2028 } 2029 }; 2030 2031 static void gic_enable_nmi_support(void) 2032 { 2033 int i; 2034 2035 if (!gic_prio_masking_enabled() || nmi_support_forbidden) 2036 return; 2037 2038 rdist_nmi_refs = kcalloc(gic_data.ppi_nr + SGI_NR, 2039 sizeof(*rdist_nmi_refs), GFP_KERNEL); 2040 if (!rdist_nmi_refs) 2041 return; 2042 2043 for (i = 0; i < gic_data.ppi_nr + SGI_NR; i++) 2044 refcount_set(&rdist_nmi_refs[i], 0); 2045 2046 pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", 2047 gic_has_relaxed_pmr_sync() ? "relaxed" : "forced"); 2048 2049 static_branch_enable(&supports_pseudo_nmis); 2050 2051 if (static_branch_likely(&supports_deactivate_key)) 2052 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 2053 else 2054 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 2055 } 2056 2057 static int __init gic_init_bases(phys_addr_t dist_phys_base, 2058 void __iomem *dist_base, 2059 struct redist_region *rdist_regs, 2060 u32 nr_redist_regions, 2061 u64 redist_stride, 2062 struct fwnode_handle *handle) 2063 { 2064 u32 typer; 2065 int err; 2066 2067 if (!is_hyp_mode_available()) 2068 static_branch_disable(&supports_deactivate_key); 2069 2070 if (static_branch_likely(&supports_deactivate_key)) 2071 pr_info("GIC: Using split EOI/Deactivate mode\n"); 2072 2073 gic_data.fwnode = handle; 2074 gic_data.dist_phys_base = dist_phys_base; 2075 gic_data.dist_base = dist_base; 2076 gic_data.redist_regions = rdist_regs; 2077 gic_data.nr_redist_regions = nr_redist_regions; 2078 gic_data.redist_stride = redist_stride; 2079 2080 /* 2081 * Find out how many interrupts are supported. 2082 */ 2083 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 2084 gic_data.rdists.gicd_typer = typer; 2085 2086 gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR), 2087 gic_quirks, &gic_data); 2088 2089 pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32); 2090 pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR); 2091 2092 /* 2093 * ThunderX1 explodes on reading GICD_TYPER2, in violation of the 2094 * architecture spec (which says that reserved registers are RES0). 2095 */ 2096 if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539)) 2097 gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2); 2098 2099 gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 2100 &gic_data); 2101 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 2102 if (!static_branch_unlikely(&gic_nvidia_t241_erratum)) { 2103 /* Disable GICv4.x features for the erratum T241-FABRIC-4 */ 2104 gic_data.rdists.has_rvpeid = true; 2105 gic_data.rdists.has_vlpis = true; 2106 gic_data.rdists.has_direct_lpi = true; 2107 gic_data.rdists.has_vpend_valid_dirty = true; 2108 } 2109 2110 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 2111 err = -ENOMEM; 2112 goto out_free; 2113 } 2114 2115 irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 2116 2117 gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 2118 2119 if (typer & GICD_TYPER_MBIS) { 2120 err = mbi_init(handle, gic_data.domain); 2121 if (err) 2122 pr_err("Failed to initialize MBIs\n"); 2123 } 2124 2125 set_handle_irq(gic_handle_irq); 2126 2127 gic_update_rdist_properties(); 2128 2129 gic_cpu_sys_reg_enable(); 2130 gic_prio_init(); 2131 gic_dist_init(); 2132 gic_cpu_init(); 2133 gic_enable_nmi_support(); 2134 gic_smp_init(); 2135 gic_cpu_pm_init(); 2136 2137 if (gic_dist_supports_lpis()) { 2138 its_init(handle, &gic_data.rdists, gic_data.domain, dist_prio_irq); 2139 its_cpu_init(); 2140 its_lpi_memreserve_init(); 2141 } else { 2142 if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) 2143 gicv2m_init(handle, gic_data.domain); 2144 } 2145 2146 return 0; 2147 2148 out_free: 2149 if (gic_data.domain) 2150 irq_domain_remove(gic_data.domain); 2151 free_percpu(gic_data.rdists.rdist); 2152 return err; 2153 } 2154 2155 static int __init gic_validate_dist_version(void __iomem *dist_base) 2156 { 2157 u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2158 2159 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 2160 return -ENODEV; 2161 2162 return 0; 2163 } 2164 2165 /* Create all possible partitions at boot time */ 2166 static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 2167 { 2168 struct device_node *parts_node, *child_part; 2169 int part_idx = 0, i; 2170 int nr_parts; 2171 struct partition_affinity *parts; 2172 2173 parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 2174 if (!parts_node) 2175 return; 2176 2177 gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); 2178 if (!gic_data.ppi_descs) 2179 goto out_put_node; 2180 2181 nr_parts = of_get_child_count(parts_node); 2182 2183 if (!nr_parts) 2184 goto out_put_node; 2185 2186 parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 2187 if (WARN_ON(!parts)) 2188 goto out_put_node; 2189 2190 for_each_child_of_node(parts_node, child_part) { 2191 struct partition_affinity *part; 2192 int n; 2193 2194 part = &parts[part_idx]; 2195 2196 part->partition_id = of_fwnode_handle(child_part); 2197 2198 pr_info("GIC: PPI partition %pOFn[%d] { ", 2199 child_part, part_idx); 2200 2201 n = of_property_count_elems_of_size(child_part, "affinity", 2202 sizeof(u32)); 2203 WARN_ON(n <= 0); 2204 2205 for (i = 0; i < n; i++) { 2206 int err, cpu; 2207 u32 cpu_phandle; 2208 struct device_node *cpu_node; 2209 2210 err = of_property_read_u32_index(child_part, "affinity", 2211 i, &cpu_phandle); 2212 if (WARN_ON(err)) 2213 continue; 2214 2215 cpu_node = of_find_node_by_phandle(cpu_phandle); 2216 if (WARN_ON(!cpu_node)) 2217 continue; 2218 2219 cpu = of_cpu_node_to_id(cpu_node); 2220 if (WARN_ON(cpu < 0)) { 2221 of_node_put(cpu_node); 2222 continue; 2223 } 2224 2225 pr_cont("%pOF[%d] ", cpu_node, cpu); 2226 2227 cpumask_set_cpu(cpu, &part->mask); 2228 of_node_put(cpu_node); 2229 } 2230 2231 pr_cont("}\n"); 2232 part_idx++; 2233 } 2234 2235 for (i = 0; i < gic_data.ppi_nr; i++) { 2236 unsigned int irq; 2237 struct partition_desc *desc; 2238 struct irq_fwspec ppi_fwspec = { 2239 .fwnode = gic_data.fwnode, 2240 .param_count = 3, 2241 .param = { 2242 [0] = GIC_IRQ_TYPE_PARTITION, 2243 [1] = i, 2244 [2] = IRQ_TYPE_NONE, 2245 }, 2246 }; 2247 2248 irq = irq_create_fwspec_mapping(&ppi_fwspec); 2249 if (WARN_ON(!irq)) 2250 continue; 2251 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 2252 irq, &partition_domain_ops); 2253 if (WARN_ON(!desc)) 2254 continue; 2255 2256 gic_data.ppi_descs[i] = desc; 2257 } 2258 2259 out_put_node: 2260 of_node_put(parts_node); 2261 } 2262 2263 static void __init gic_of_setup_kvm_info(struct device_node *node, u32 nr_redist_regions) 2264 { 2265 int ret; 2266 struct resource r; 2267 2268 gic_v3_kvm_info.type = GIC_V3; 2269 2270 gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 2271 if (!gic_v3_kvm_info.maint_irq) 2272 return; 2273 2274 /* Also skip GICD, GICC, GICH */ 2275 ret = of_address_to_resource(node, nr_redist_regions + 3, &r); 2276 if (!ret) 2277 gic_v3_kvm_info.vcpu = r; 2278 2279 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 2280 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 2281 vgic_set_kvm_info(&gic_v3_kvm_info); 2282 } 2283 2284 static void gic_request_region(resource_size_t base, resource_size_t size, 2285 const char *name) 2286 { 2287 if (!request_mem_region(base, size, name)) 2288 pr_warn_once(FW_BUG "%s region %pa has overlapping address\n", 2289 name, &base); 2290 } 2291 2292 static void __iomem *gic_of_iomap(struct device_node *node, int idx, 2293 const char *name, struct resource *res) 2294 { 2295 void __iomem *base; 2296 int ret; 2297 2298 ret = of_address_to_resource(node, idx, res); 2299 if (ret) 2300 return IOMEM_ERR_PTR(ret); 2301 2302 gic_request_region(res->start, resource_size(res), name); 2303 base = of_iomap(node, idx); 2304 2305 return base ?: IOMEM_ERR_PTR(-ENOMEM); 2306 } 2307 2308 static int __init gic_of_init(struct device_node *node, struct device_node *parent) 2309 { 2310 phys_addr_t dist_phys_base; 2311 void __iomem *dist_base; 2312 struct redist_region *rdist_regs; 2313 struct resource res; 2314 u64 redist_stride; 2315 u32 nr_redist_regions; 2316 int err, i; 2317 2318 dist_base = gic_of_iomap(node, 0, "GICD", &res); 2319 if (IS_ERR(dist_base)) { 2320 pr_err("%pOF: unable to map gic dist registers\n", node); 2321 return PTR_ERR(dist_base); 2322 } 2323 2324 dist_phys_base = res.start; 2325 2326 err = gic_validate_dist_version(dist_base); 2327 if (err) { 2328 pr_err("%pOF: no distributor detected, giving up\n", node); 2329 goto out_unmap_dist; 2330 } 2331 2332 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 2333 nr_redist_regions = 1; 2334 2335 rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 2336 GFP_KERNEL); 2337 if (!rdist_regs) { 2338 err = -ENOMEM; 2339 goto out_unmap_dist; 2340 } 2341 2342 for (i = 0; i < nr_redist_regions; i++) { 2343 rdist_regs[i].redist_base = gic_of_iomap(node, 1 + i, "GICR", &res); 2344 if (IS_ERR(rdist_regs[i].redist_base)) { 2345 pr_err("%pOF: couldn't map region %d\n", node, i); 2346 err = -ENODEV; 2347 goto out_unmap_rdist; 2348 } 2349 rdist_regs[i].phys_base = res.start; 2350 } 2351 2352 if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 2353 redist_stride = 0; 2354 2355 gic_enable_of_quirks(node, gic_quirks, &gic_data); 2356 2357 err = gic_init_bases(dist_phys_base, dist_base, rdist_regs, 2358 nr_redist_regions, redist_stride, &node->fwnode); 2359 if (err) 2360 goto out_unmap_rdist; 2361 2362 gic_populate_ppi_partitions(node); 2363 2364 if (static_branch_likely(&supports_deactivate_key)) 2365 gic_of_setup_kvm_info(node, nr_redist_regions); 2366 return 0; 2367 2368 out_unmap_rdist: 2369 for (i = 0; i < nr_redist_regions; i++) 2370 if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base)) 2371 iounmap(rdist_regs[i].redist_base); 2372 kfree(rdist_regs); 2373 out_unmap_dist: 2374 iounmap(dist_base); 2375 return err; 2376 } 2377 2378 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 2379 2380 #ifdef CONFIG_ACPI 2381 static struct 2382 { 2383 void __iomem *dist_base; 2384 struct redist_region *redist_regs; 2385 u32 nr_redist_regions; 2386 bool single_redist; 2387 int enabled_rdists; 2388 u32 maint_irq; 2389 int maint_irq_mode; 2390 phys_addr_t vcpu_base; 2391 } acpi_data __initdata; 2392 2393 static void __init 2394 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 2395 { 2396 static int count = 0; 2397 2398 acpi_data.redist_regs[count].phys_base = phys_base; 2399 acpi_data.redist_regs[count].redist_base = redist_base; 2400 acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 2401 count++; 2402 } 2403 2404 static int __init 2405 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 2406 const unsigned long end) 2407 { 2408 struct acpi_madt_generic_redistributor *redist = 2409 (struct acpi_madt_generic_redistributor *)header; 2410 void __iomem *redist_base; 2411 2412 redist_base = ioremap(redist->base_address, redist->length); 2413 if (!redist_base) { 2414 pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 2415 return -ENOMEM; 2416 } 2417 2418 if (acpi_get_madt_revision() >= 7 && 2419 (redist->flags & ACPI_MADT_GICR_NON_COHERENT)) 2420 gic_data.rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; 2421 2422 gic_request_region(redist->base_address, redist->length, "GICR"); 2423 2424 gic_acpi_register_redist(redist->base_address, redist_base); 2425 return 0; 2426 } 2427 2428 static int __init 2429 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 2430 const unsigned long end) 2431 { 2432 struct acpi_madt_generic_interrupt *gicc = 2433 (struct acpi_madt_generic_interrupt *)header; 2434 u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 2435 u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 2436 void __iomem *redist_base; 2437 2438 /* Neither enabled or online capable means it doesn't exist, skip it */ 2439 if (!(gicc->flags & (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) 2440 return 0; 2441 2442 /* 2443 * Capable but disabled CPUs can be brought online later. What about 2444 * the redistributor? ACPI doesn't want to say! 2445 * Virtual hotplug systems can use the MADT's "always-on" GICR entries. 2446 * Otherwise, prevent such CPUs from being brought online. 2447 */ 2448 if (!(gicc->flags & ACPI_MADT_ENABLED)) { 2449 int cpu = get_cpu_for_acpi_id(gicc->uid); 2450 2451 pr_warn("CPU %u's redistributor is inaccessible: this CPU can't be brought online\n", cpu); 2452 if (cpu >= 0) 2453 cpumask_set_cpu(cpu, &broken_rdists); 2454 return 0; 2455 } 2456 2457 redist_base = ioremap(gicc->gicr_base_address, size); 2458 if (!redist_base) 2459 return -ENOMEM; 2460 gic_request_region(gicc->gicr_base_address, size, "GICR"); 2461 2462 if (acpi_get_madt_revision() >= 7 && 2463 (gicc->flags & ACPI_MADT_GICC_NON_COHERENT)) 2464 gic_data.rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; 2465 2466 gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 2467 return 0; 2468 } 2469 2470 static int __init gic_acpi_collect_gicr_base(void) 2471 { 2472 acpi_tbl_entry_handler redist_parser; 2473 enum acpi_madt_type type; 2474 2475 if (acpi_data.single_redist) { 2476 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 2477 redist_parser = gic_acpi_parse_madt_gicc; 2478 } else { 2479 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 2480 redist_parser = gic_acpi_parse_madt_redist; 2481 } 2482 2483 /* Collect redistributor base addresses in GICR entries */ 2484 if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 2485 return 0; 2486 2487 pr_info("No valid GICR entries exist\n"); 2488 return -ENODEV; 2489 } 2490 2491 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 2492 const unsigned long end) 2493 { 2494 /* Subtable presence means that redist exists, that's it */ 2495 return 0; 2496 } 2497 2498 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 2499 const unsigned long end) 2500 { 2501 struct acpi_madt_generic_interrupt *gicc = 2502 (struct acpi_madt_generic_interrupt *)header; 2503 2504 /* 2505 * If GICC is enabled and has valid gicr base address, then it means 2506 * GICR base is presented via GICC. The redistributor is only known to 2507 * be accessible if the GICC is marked as enabled. If this bit is not 2508 * set, we'd need to add the redistributor at runtime, which isn't 2509 * supported. 2510 */ 2511 if (gicc->flags & ACPI_MADT_ENABLED && gicc->gicr_base_address) 2512 acpi_data.enabled_rdists++; 2513 2514 return 0; 2515 } 2516 2517 static int __init gic_acpi_count_gicr_regions(void) 2518 { 2519 int count; 2520 2521 /* 2522 * Count how many redistributor regions we have. It is not allowed 2523 * to mix redistributor description, GICR and GICC subtables have to be 2524 * mutually exclusive. 2525 */ 2526 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 2527 gic_acpi_match_gicr, 0); 2528 if (count > 0) { 2529 acpi_data.single_redist = false; 2530 return count; 2531 } 2532 2533 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2534 gic_acpi_match_gicc, 0); 2535 if (count > 0) { 2536 acpi_data.single_redist = true; 2537 count = acpi_data.enabled_rdists; 2538 } 2539 2540 return count; 2541 } 2542 2543 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 2544 struct acpi_probe_entry *ape) 2545 { 2546 struct acpi_madt_generic_distributor *dist; 2547 int count; 2548 2549 dist = (struct acpi_madt_generic_distributor *)header; 2550 if (dist->version != ape->driver_data) 2551 return false; 2552 2553 /* We need to do that exercise anyway, the sooner the better */ 2554 count = gic_acpi_count_gicr_regions(); 2555 if (count <= 0) 2556 return false; 2557 2558 acpi_data.nr_redist_regions = count; 2559 return true; 2560 } 2561 2562 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 2563 const unsigned long end) 2564 { 2565 struct acpi_madt_generic_interrupt *gicc = 2566 (struct acpi_madt_generic_interrupt *)header; 2567 int maint_irq_mode; 2568 static int first_madt = true; 2569 2570 if (!(gicc->flags & 2571 (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) 2572 return 0; 2573 2574 maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 2575 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 2576 2577 if (first_madt) { 2578 first_madt = false; 2579 2580 acpi_data.maint_irq = gicc->vgic_interrupt; 2581 acpi_data.maint_irq_mode = maint_irq_mode; 2582 acpi_data.vcpu_base = gicc->gicv_base_address; 2583 2584 return 0; 2585 } 2586 2587 /* 2588 * The maintenance interrupt and GICV should be the same for every CPU 2589 */ 2590 if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 2591 (acpi_data.maint_irq_mode != maint_irq_mode) || 2592 (acpi_data.vcpu_base != gicc->gicv_base_address)) 2593 return -EINVAL; 2594 2595 return 0; 2596 } 2597 2598 static bool __init gic_acpi_collect_virt_info(void) 2599 { 2600 int count; 2601 2602 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 2603 gic_acpi_parse_virt_madt_gicc, 0); 2604 2605 return (count > 0); 2606 } 2607 2608 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 2609 #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 2610 #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 2611 2612 static void __init gic_acpi_setup_kvm_info(void) 2613 { 2614 int irq; 2615 2616 if (!gic_acpi_collect_virt_info()) { 2617 pr_warn("Unable to get hardware information used for virtualization\n"); 2618 return; 2619 } 2620 2621 gic_v3_kvm_info.type = GIC_V3; 2622 2623 irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 2624 acpi_data.maint_irq_mode, 2625 ACPI_ACTIVE_HIGH); 2626 if (irq <= 0) 2627 return; 2628 2629 gic_v3_kvm_info.maint_irq = irq; 2630 2631 if (acpi_data.vcpu_base) { 2632 struct resource *vcpu = &gic_v3_kvm_info.vcpu; 2633 2634 vcpu->flags = IORESOURCE_MEM; 2635 vcpu->start = acpi_data.vcpu_base; 2636 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 2637 } 2638 2639 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 2640 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; 2641 vgic_set_kvm_info(&gic_v3_kvm_info); 2642 } 2643 2644 static struct fwnode_handle *gsi_domain_handle; 2645 2646 static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi) 2647 { 2648 return gsi_domain_handle; 2649 } 2650 2651 static int __init 2652 gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end) 2653 { 2654 struct acpi_madt_generic_distributor *dist; 2655 size_t size; 2656 int i, err; 2657 2658 /* Get distributor base address */ 2659 dist = (struct acpi_madt_generic_distributor *)header; 2660 acpi_data.dist_base = ioremap(dist->base_address, 2661 ACPI_GICV3_DIST_MEM_SIZE); 2662 if (!acpi_data.dist_base) { 2663 pr_err("Unable to map GICD registers\n"); 2664 return -ENOMEM; 2665 } 2666 gic_request_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD"); 2667 2668 err = gic_validate_dist_version(acpi_data.dist_base); 2669 if (err) { 2670 pr_err("No distributor detected at @%p, giving up\n", 2671 acpi_data.dist_base); 2672 goto out_dist_unmap; 2673 } 2674 2675 size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 2676 acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 2677 if (!acpi_data.redist_regs) { 2678 err = -ENOMEM; 2679 goto out_dist_unmap; 2680 } 2681 2682 err = gic_acpi_collect_gicr_base(); 2683 if (err) 2684 goto out_redist_unmap; 2685 2686 gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address); 2687 if (!gsi_domain_handle) { 2688 err = -ENOMEM; 2689 goto out_redist_unmap; 2690 } 2691 2692 err = gic_init_bases(dist->base_address, acpi_data.dist_base, 2693 acpi_data.redist_regs, acpi_data.nr_redist_regions, 2694 0, gsi_domain_handle); 2695 if (err) 2696 goto out_fwhandle_free; 2697 2698 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id); 2699 2700 if (static_branch_likely(&supports_deactivate_key)) 2701 gic_acpi_setup_kvm_info(); 2702 2703 return 0; 2704 2705 out_fwhandle_free: 2706 irq_domain_free_fwnode(gsi_domain_handle); 2707 out_redist_unmap: 2708 for (i = 0; i < acpi_data.nr_redist_regions; i++) 2709 if (acpi_data.redist_regs[i].redist_base) 2710 iounmap(acpi_data.redist_regs[i].redist_base); 2711 kfree(acpi_data.redist_regs); 2712 out_dist_unmap: 2713 iounmap(acpi_data.dist_base); 2714 return err; 2715 } 2716 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2717 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 2718 gic_acpi_init); 2719 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2720 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 2721 gic_acpi_init); 2722 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 2723 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 2724 gic_acpi_init); 2725 #endif