From c5e5ec033c4ab25c53f1fd217849e75deb0bf7bf Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 May 2020 10:41:00 +0100 Subject: [PATCH 01/18] genirq: Add fasteoi IPI flow For irqchips using the fasteoi flow, IPIs are a bit special. They need to be EOI'd early (before calling the handler), as funny things may happen in the handler (they do not necessarily behave like a normal interrupt). Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- include/linux/irq.h | 1 + kernel/irq/chip.c | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/include/linux/irq.h b/include/linux/irq.h index 1b7f4dfee35b..57205bbf46bf 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -634,6 +634,7 @@ static inline int irq_set_parent(int irq, int parent_irq) */ extern void handle_level_irq(struct irq_desc *desc); extern void handle_fasteoi_irq(struct irq_desc *desc); +extern void handle_percpu_devid_fasteoi_ipi(struct irq_desc *desc); extern void handle_edge_irq(struct irq_desc *desc); extern void handle_edge_eoi_irq(struct irq_desc *desc); extern void handle_simple_irq(struct irq_desc *desc); diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 857f5f4c8098..bed517d5aa4d 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -944,6 +944,33 @@ void handle_percpu_devid_irq(struct irq_desc *desc) chip->irq_eoi(&desc->irq_data); } +/** + * handle_percpu_devid_fasteoi_ipi - Per CPU local IPI handler with per cpu + * dev ids + * @desc: the interrupt description structure for this irq + * + * The biggest difference with the IRQ version is that the interrupt is + * EOIed early, as the IPI could result in a context switch, and we need to + * make sure the IPI can fire again. We also assume that the arch code has + * registered an action. If not, we are positively doomed. + */ +void handle_percpu_devid_fasteoi_ipi(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct irqaction *action = desc->action; + unsigned int irq = irq_desc_get_irq(desc); + irqreturn_t res; + + __kstat_incr_irqs_this_cpu(desc); + + if (chip->irq_eoi) + chip->irq_eoi(&desc->irq_data); + + trace_irq_handler_entry(irq, action); + res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id)); + trace_irq_handler_exit(irq, action, res); +} + /** * handle_percpu_devid_fasteoi_nmi - Per CPU local NMI handler with per cpu * dev ids From 83cfac95c01817819c2a51f0931d798d851f8a08 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 May 2020 14:58:13 +0100 Subject: [PATCH 02/18] genirq: Allow interrupts to be excluded from /proc/interrupts A number of architectures implement IPI statistics directly, duplicating the core kstat_irqs accounting. As we move IPIs to being actual IRQs, we would end-up with a confusing display in /proc/interrupts (where the IPIs would appear twice). In order to solve this, allow interrupts to be flagged as "hidden", which excludes them from /proc/interrupts. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- include/linux/irq.h | 4 +++- kernel/irq/debugfs.c | 1 + kernel/irq/proc.c | 2 +- kernel/irq/settings.h | 7 +++++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/include/linux/irq.h b/include/linux/irq.h index 57205bbf46bf..63b9d962ee67 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -71,6 +71,7 @@ enum irqchip_irq_state; * it from the spurious interrupt detection * mechanism and from core side polling. * IRQ_DISABLE_UNLAZY - Disable lazy irq disable + * IRQ_HIDDEN - Don't show up in /proc/interrupts */ enum { IRQ_TYPE_NONE = 0x00000000, @@ -97,13 +98,14 @@ enum { IRQ_PER_CPU_DEVID = (1 << 17), IRQ_IS_POLLED = (1 << 18), IRQ_DISABLE_UNLAZY = (1 << 19), + IRQ_HIDDEN = (1 << 20), }; #define IRQF_MODIFY_MASK \ (IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \ IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \ IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \ - IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY) + IRQ_IS_POLLED | IRQ_DISABLE_UNLAZY | IRQ_HIDDEN) #define IRQ_NO_BALANCING_MASK (IRQ_PER_CPU | IRQ_NO_BALANCING) diff --git a/kernel/irq/debugfs.c b/kernel/irq/debugfs.c index b95ff5d5f4bd..acabc0c0e46b 100644 --- a/kernel/irq/debugfs.c +++ b/kernel/irq/debugfs.c @@ -136,6 +136,7 @@ static const struct irq_bit_descr irqdesc_states[] = { BIT_MASK_DESCR(_IRQ_PER_CPU_DEVID), BIT_MASK_DESCR(_IRQ_IS_POLLED), BIT_MASK_DESCR(_IRQ_DISABLE_UNLAZY), + BIT_MASK_DESCR(_IRQ_HIDDEN), }; static const struct irq_bit_descr irqdesc_istates[] = { diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 32c071d7bc03..72513ed2a5fc 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -485,7 +485,7 @@ int show_interrupts(struct seq_file *p, void *v) rcu_read_lock(); desc = irq_to_desc(i); - if (!desc) + if (!desc || irq_settings_is_hidden(desc)) goto outsparse; if (desc->kstat_irqs) diff --git a/kernel/irq/settings.h b/kernel/irq/settings.h index e43795cd2ccf..403378b9947b 100644 --- a/kernel/irq/settings.h +++ b/kernel/irq/settings.h @@ -17,6 +17,7 @@ enum { _IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID, _IRQ_IS_POLLED = IRQ_IS_POLLED, _IRQ_DISABLE_UNLAZY = IRQ_DISABLE_UNLAZY, + _IRQ_HIDDEN = IRQ_HIDDEN, _IRQF_MODIFY_MASK = IRQF_MODIFY_MASK, }; @@ -31,6 +32,7 @@ enum { #define IRQ_PER_CPU_DEVID GOT_YOU_MORON #define IRQ_IS_POLLED GOT_YOU_MORON #define IRQ_DISABLE_UNLAZY GOT_YOU_MORON +#define IRQ_HIDDEN GOT_YOU_MORON #undef IRQF_MODIFY_MASK #define IRQF_MODIFY_MASK GOT_YOU_MORON @@ -167,3 +169,8 @@ static inline void irq_settings_clr_disable_unlazy(struct irq_desc *desc) { desc->status_use_accessors &= ~_IRQ_DISABLE_UNLAZY; } + +static inline bool irq_settings_is_hidden(struct irq_desc *desc) +{ + return desc->status_use_accessors & _IRQ_HIDDEN; +} From d3afc7f12987581eb0d1215b518d719fb9d762da Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 25 Apr 2020 15:03:47 +0100 Subject: [PATCH 03/18] arm64: Allow IPIs to be handled as normal interrupts In order to deal with IPIs as normal interrupts, let's add a new way to register them with the architecture code. set_smp_ipi_range() takes a range of interrupts, and allows the arch code to request them as if the were normal interrupts. A standard handler is then called by the core IRQ code to deal with the IPI. This means that we don't need to call irq_enter/irq_exit, and that we don't need to deal with set_irq_regs either. So let's move the dispatcher into its own function, and leave handle_IPI() as a compatibility function. On the sending side, let's make use of ipi_send_mask, which already exists for this purpose. One of the major difference is that we end up, in some cases (such as when performing IRQ time accounting on the scheduler IPI), end up with nested irq_enter()/irq_exit() pairs. Other than the (relatively small) overhead, there should be no consequences to it (these pairs are designed to nest correctly, and the accounting shouldn't be off). Reviewed-by: Valentin Schneider Acked-by: Catalin Marinas Signed-off-by: Marc Zyngier --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/smp.h | 5 ++ arch/arm64/kernel/smp.c | 93 +++++++++++++++++++++++++++++++----- 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6d232837cbee..d0fdbe5fb32f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -106,6 +106,7 @@ config ARM64 select GENERIC_CPU_VULNERABILITIES select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP + select GENERIC_IRQ_IPI select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 0eadbf933e35..57c5db15f6b7 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -78,6 +78,11 @@ extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); extern void (*__smp_cross_call)(const struct cpumask *, unsigned int); +/* + * Register IPI interrupts with the arch SMP code + */ +extern void set_smp_ipi_range(int ipi_base, int nr_ipi); + /* * Called from the secondary holding pen, this is the secondary CPU entry point. */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 355ee9eed4dd..00c9db1b61b5 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -75,6 +75,13 @@ enum ipi_msg_type { IPI_WAKEUP }; +static int ipi_irq_base __read_mostly; +static int nr_ipi __read_mostly = NR_IPI; +static struct irq_desc *ipi_desc[NR_IPI] __read_mostly; + +static void ipi_setup(int cpu); +static void ipi_teardown(int cpu); + #ifdef CONFIG_HOTPLUG_CPU static int op_cpu_kill(unsigned int cpu); #else @@ -237,6 +244,8 @@ asmlinkage notrace void secondary_start_kernel(void) */ notify_cpu_starting(cpu); + ipi_setup(cpu); + store_cpu_topology(cpu); numa_add_cpu(cpu); @@ -302,6 +311,7 @@ int __cpu_disable(void) * and we must not schedule until we're ready to give up the cpu. */ set_cpu_online(cpu, false); + ipi_teardown(cpu); /* * OK - migrate IRQs away from this CPU @@ -890,10 +900,9 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) /* * Main handler for inter-processor interrupts */ -void handle_IPI(int ipinr, struct pt_regs *regs) +static void do_handle_IPI(int ipinr) { unsigned int cpu = smp_processor_id(); - struct pt_regs *old_regs = set_irq_regs(regs); if ((unsigned)ipinr < NR_IPI) { trace_ipi_entry_rcuidle(ipi_types[ipinr]); @@ -906,21 +915,16 @@ void handle_IPI(int ipinr, struct pt_regs *regs) break; case IPI_CALL_FUNC: - irq_enter(); generic_smp_call_function_interrupt(); - irq_exit(); break; case IPI_CPU_STOP: - irq_enter(); local_cpu_stop(); - irq_exit(); break; case IPI_CPU_CRASH_STOP: if (IS_ENABLED(CONFIG_KEXEC_CORE)) { - irq_enter(); - ipi_cpu_crash_stop(cpu, regs); + ipi_cpu_crash_stop(cpu, get_irq_regs()); unreachable(); } @@ -928,17 +932,13 @@ void handle_IPI(int ipinr, struct pt_regs *regs) #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST case IPI_TIMER: - irq_enter(); tick_receive_broadcast(); - irq_exit(); break; #endif #ifdef CONFIG_IRQ_WORK case IPI_IRQ_WORK: - irq_enter(); irq_work_run(); - irq_exit(); break; #endif @@ -957,9 +957,78 @@ void handle_IPI(int ipinr, struct pt_regs *regs) if ((unsigned)ipinr < NR_IPI) trace_ipi_exit_rcuidle(ipi_types[ipinr]); +} + +/* Legacy version, should go away once all irqchips have been converted */ +void handle_IPI(int ipinr, struct pt_regs *regs) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + + irq_enter(); + do_handle_IPI(ipinr); + irq_exit(); + set_irq_regs(old_regs); } +static irqreturn_t ipi_handler(int irq, void *data) +{ + do_handle_IPI(irq - ipi_irq_base); + return IRQ_HANDLED; +} + +static void ipi_send(const struct cpumask *target, unsigned int ipi) +{ + __ipi_send_mask(ipi_desc[ipi], target); +} + +static void ipi_setup(int cpu) +{ + int i; + + if (!ipi_irq_base) + return; + + for (i = 0; i < nr_ipi; i++) + enable_percpu_irq(ipi_irq_base + i, 0); +} + +static void ipi_teardown(int cpu) +{ + int i; + + if (!ipi_irq_base) + return; + + for (i = 0; i < nr_ipi; i++) + disable_percpu_irq(ipi_irq_base + i); +} + +void __init set_smp_ipi_range(int ipi_base, int n) +{ + int i; + + WARN_ON(n < NR_IPI); + nr_ipi = min(n, NR_IPI); + + for (i = 0; i < nr_ipi; i++) { + int err; + + err = request_percpu_irq(ipi_base + i, ipi_handler, + "IPI", &irq_stat); + WARN_ON(err); + + ipi_desc[i] = irq_to_desc(ipi_base + i); + irq_set_status_flags(ipi_base + i, IRQ_HIDDEN); + } + + ipi_irq_base = ipi_base; + __smp_cross_call = ipi_send; + + /* Setup the boot CPU immediately */ + ipi_setup(smp_processor_id()); +} + void smp_send_reschedule(int cpu) { smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); From 56afcd3dbd1995c526bfbd920cebde6158b22c4a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 23 Jun 2020 20:38:41 +0100 Subject: [PATCH 04/18] ARM: Allow IPIs to be handled as normal interrupts In order to deal with IPIs as normal interrupts, let's add a new way to register them with the architecture code. set_smp_ipi_range() takes a range of interrupts, and allows the arch code to request them as if the were normal interrupts. A standard handler is then called by the core IRQ code to deal with the IPI. This means that we don't need to call irq_enter/irq_exit, and that we don't need to deal with set_irq_regs either. So let's move the dispatcher into its own function, and leave handle_IPI() as a compatibility function. On the sending side, let's make use of ipi_send_mask, which already exists for this purpose. One of the major difference is that we end up, in some cases (such as when performing IRQ time accounting on the scheduler IPI), end up with nested irq_enter()/irq_exit() pairs. Other than the (relatively small) overhead, there should be no consequences to it (these pairs are designed to nest correctly, and the accounting shouldn't be off). Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- arch/arm/Kconfig | 1 + arch/arm/include/asm/smp.h | 5 ++ arch/arm/kernel/smp.c | 99 ++++++++++++++++++++++++++++++++------ 3 files changed, 89 insertions(+), 16 deletions(-) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e00d94b16658..e67ef15c800f 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -49,6 +49,7 @@ config ARM select GENERIC_ARCH_TOPOLOGY if ARM_CPU_TOPOLOGY select GENERIC_ATOMIC64 if CPU_V7M || CPU_V6 || !CPU_32v6K || !AEABI select GENERIC_CLOCKEVENTS_BROADCAST if SMP + select GENERIC_IRQ_IPI if SMP select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index a91f21e3c5b5..0e29730295ca 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -45,6 +45,11 @@ extern void smp_init_cpus(void); */ extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); +/* + * Register IPI interrupts with the arch SMP code + */ +extern void set_smp_ipi_range(int ipi_base, int nr_ipi); + /* * Called from platform specific assembly code, this is the * secondary CPU entry point. diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 5d9da61eff62..f21f78483353 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -69,14 +69,22 @@ enum ipi_msg_type { * CPU_BACKTRACE is special and not included in NR_IPI * or tracable with trace_ipi_* */ - IPI_CPU_BACKTRACE, + IPI_CPU_BACKTRACE = NR_IPI, /* * SGI8-15 can be reserved by secure firmware, and thus may * not be usable by the kernel. Please keep the above limited * to at most 8 entries. */ + MAX_IPI }; +static int ipi_irq_base __read_mostly; +static int nr_ipi __read_mostly = NR_IPI; +static struct irq_desc *ipi_desc[MAX_IPI] __read_mostly; + +static void ipi_setup(int cpu); +static void ipi_teardown(int cpu); + static DECLARE_COMPLETION(cpu_running); static struct smp_operations smp_ops __ro_after_init; @@ -247,6 +255,7 @@ int __cpu_disable(void) * and we must not schedule until we're ready to give up the cpu. */ set_cpu_online(cpu, false); + ipi_teardown(cpu); /* * OK - migrate IRQs away from this CPU @@ -422,6 +431,8 @@ asmlinkage void secondary_start_kernel(void) notify_cpu_starting(cpu); + ipi_setup(cpu); + calibrate_delay(); smp_store_cpu_info(cpu); @@ -627,10 +638,9 @@ asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs) handle_IPI(ipinr, regs); } -void handle_IPI(int ipinr, struct pt_regs *regs) +static void do_handle_IPI(int ipinr) { unsigned int cpu = smp_processor_id(); - struct pt_regs *old_regs = set_irq_regs(regs); if ((unsigned)ipinr < NR_IPI) { trace_ipi_entry_rcuidle(ipi_types[ipinr]); @@ -643,9 +653,7 @@ void handle_IPI(int ipinr, struct pt_regs *regs) #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST case IPI_TIMER: - irq_enter(); tick_receive_broadcast(); - irq_exit(); break; #endif @@ -654,36 +662,26 @@ void handle_IPI(int ipinr, struct pt_regs *regs) break; case IPI_CALL_FUNC: - irq_enter(); generic_smp_call_function_interrupt(); - irq_exit(); break; case IPI_CPU_STOP: - irq_enter(); ipi_cpu_stop(cpu); - irq_exit(); break; #ifdef CONFIG_IRQ_WORK case IPI_IRQ_WORK: - irq_enter(); irq_work_run(); - irq_exit(); break; #endif case IPI_COMPLETION: - irq_enter(); ipi_complete(cpu); - irq_exit(); break; case IPI_CPU_BACKTRACE: printk_nmi_enter(); - irq_enter(); - nmi_cpu_backtrace(regs); - irq_exit(); + nmi_cpu_backtrace(get_irq_regs()); printk_nmi_exit(); break; @@ -695,9 +693,78 @@ void handle_IPI(int ipinr, struct pt_regs *regs) if ((unsigned)ipinr < NR_IPI) trace_ipi_exit_rcuidle(ipi_types[ipinr]); +} + +/* Legacy version, should go away once all irqchips have been converted */ +void handle_IPI(int ipinr, struct pt_regs *regs) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + + irq_enter(); + do_handle_IPI(ipinr); + irq_exit(); + set_irq_regs(old_regs); } +static irqreturn_t ipi_handler(int irq, void *data) +{ + do_handle_IPI(irq - ipi_irq_base); + return IRQ_HANDLED; +} + +static void ipi_send(const struct cpumask *target, unsigned int ipi) +{ + __ipi_send_mask(ipi_desc[ipi], target); +} + +static void ipi_setup(int cpu) +{ + int i; + + if (!ipi_irq_base) + return; + + for (i = 0; i < nr_ipi; i++) + enable_percpu_irq(ipi_irq_base + i, 0); +} + +static void ipi_teardown(int cpu) +{ + int i; + + if (!ipi_irq_base) + return; + + for (i = 0; i < nr_ipi; i++) + disable_percpu_irq(ipi_irq_base + i); +} + +void __init set_smp_ipi_range(int ipi_base, int n) +{ + int i; + + WARN_ON(n < MAX_IPI); + nr_ipi = min(n, MAX_IPI); + + for (i = 0; i < nr_ipi; i++) { + int err; + + err = request_percpu_irq(ipi_base + i, ipi_handler, + "IPI", &irq_stat); + WARN_ON(err); + + ipi_desc[i] = irq_to_desc(ipi_base + i); + irq_set_status_flags(ipi_base + i, IRQ_HIDDEN); + } + + ipi_irq_base = ipi_base; + set_smp_cross_call(ipi_send); + + /* Setup the boot CPU immediately */ + ipi_setup(smp_processor_id()); +} + void smp_send_reschedule(int cpu) { smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); From 70a29c32cf7909e96a469ae71d88b2c0fbcbd767 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 25 Apr 2020 15:11:20 +0100 Subject: [PATCH 05/18] irqchip/gic-v3: Describe the SGI range As we are about to start making use of SGIs in a more conventional way, let's describe it is the GICv3 list of interrupt types. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-gic-v3.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 850842f27bee..f7c99a302d01 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -112,6 +112,7 @@ static DEFINE_PER_CPU(bool, has_rss); #define DEFAULT_PMR_VALUE 0xf0 enum gic_intid_range { + SGI_RANGE, PPI_RANGE, SPI_RANGE, EPPI_RANGE, @@ -123,6 +124,8 @@ enum gic_intid_range { static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) { switch (hwirq) { + case 0 ... 15: + return SGI_RANGE; case 16 ... 31: return PPI_RANGE; case 32 ... 1019: @@ -148,15 +151,22 @@ static inline unsigned int gic_irq(struct irq_data *d) return d->hwirq; } -static inline int gic_irq_in_rdist(struct irq_data *d) +static inline bool gic_irq_in_rdist(struct irq_data *d) { - enum gic_intid_range range = get_intid_range(d); - return range == PPI_RANGE || range == EPPI_RANGE; + switch (get_intid_range(d)) { + case SGI_RANGE: + case PPI_RANGE: + case EPPI_RANGE: + return true; + default: + return false; + } } static inline void __iomem *gic_dist_base(struct irq_data *d) { switch (get_intid_range(d)) { + case SGI_RANGE: case PPI_RANGE: case EPPI_RANGE: /* SGI+PPI -> SGI_base for this CPU */ @@ -253,6 +263,7 @@ static void gic_enable_redist(bool enable) static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index) { switch (get_intid_range(d)) { + case SGI_RANGE: case PPI_RANGE: case SPI_RANGE: *index = d->hwirq; @@ -1277,6 +1288,7 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, chip = &gic_eoimode1_chip; switch (__get_intid_range(hw)) { + case SGI_RANGE: case PPI_RANGE: case EPPI_RANGE: irq_set_percpu_devid(irq); From 64b499d8df40dadb1818ad9f74c4546951b37a8f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 25 Apr 2020 15:24:01 +0100 Subject: [PATCH 06/18] irqchip/gic-v3: Configure SGIs as standard interrupts Change the way we deal with GICv3 SGIs by turning them into proper IRQs, and calling into the arch code to register the interrupt range instead of a callback. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-gic-v3.c | 94 +++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 43 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index f7c99a302d01..84ceb6334e72 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -36,6 +36,8 @@ #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) +#define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) + struct redist_region { void __iomem *redist_base; phys_addr_t phys_base; @@ -383,7 +385,7 @@ static int gic_irq_set_irqchip_state(struct irq_data *d, { u32 reg; - if (d->hwirq >= 8192) /* PPI/SPI only */ + if (d->hwirq >= 8192) /* SGI/PPI/SPI only */ return -EINVAL; switch (which) { @@ -550,12 +552,12 @@ static int gic_set_type(struct irq_data *d, unsigned int type) u32 offset, index; int ret; - /* Interrupt configuration for SGIs can't be changed */ - if (irq < 16) - return -EINVAL; - range = get_intid_range(d); + /* Interrupt configuration for SGIs can't be changed */ + if (range == SGI_RANGE) + return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; + /* SPIs have restrictions on the supported types */ if ((range == SPI_RANGE || range == ESPI_RANGE) && type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) @@ -583,6 +585,9 @@ static int gic_set_type(struct irq_data *d, unsigned int type) static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) { + if (get_intid_range(d) == SGI_RANGE) + return -EINVAL; + if (vcpu) irqd_set_forwarded_to_vcpu(d); else @@ -657,38 +662,14 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs if ((irqnr >= 1020 && irqnr <= 1023)) return; - /* Treat anything but SGIs in a uniform way */ - if (likely(irqnr > 15)) { - int err; - - if (static_branch_likely(&supports_deactivate_key)) - gic_write_eoir(irqnr); - else - isb(); - - err = handle_domain_irq(gic_data.domain, irqnr, regs); - if (err) { - WARN_ONCE(true, "Unexpected interrupt received!\n"); - gic_deactivate_unhandled(irqnr); - } - return; - } - if (irqnr < 16) { + if (static_branch_likely(&supports_deactivate_key)) gic_write_eoir(irqnr); - if (static_branch_likely(&supports_deactivate_key)) - gic_write_dir(irqnr); -#ifdef CONFIG_SMP - /* - * Unlike GICv2, we don't need an smp_rmb() here. - * The control dependency from gic_read_iar to - * the ISB in gic_write_eoir is enough to ensure - * that any shared data read by handle_IPI will - * be read after the ACK. - */ - handle_IPI(irqnr, regs); -#else - WARN_ONCE(true, "Unexpected SGI received!\n"); -#endif + else + isb(); + + if (handle_domain_irq(gic_data.domain, irqnr, regs)) { + WARN_ONCE(true, "Unexpected interrupt received!\n"); + gic_deactivate_unhandled(irqnr); } } @@ -1136,11 +1117,11 @@ static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) gic_write_sgi1r(val); } -static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) +static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { int cpu; - if (WARN_ON(irq >= 16)) + if (WARN_ON(d->hwirq >= 16)) return; /* @@ -1154,7 +1135,7 @@ static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) u16 tlist; tlist = gic_compute_target_list(&cpu, mask, cluster_id); - gic_send_sgi(cluster_id, tlist, irq); + gic_send_sgi(cluster_id, tlist, d->hwirq); } /* Force the above writes to ICC_SGI1R_EL1 to be executed */ @@ -1163,10 +1144,24 @@ static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) static void __init gic_smp_init(void) { - set_smp_cross_call(gic_raise_softirq); + struct irq_fwspec sgi_fwspec = { + .fwnode = gic_data.fwnode, + .param_count = 1, + }; + int base_sgi; + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, "irqchip/arm/gicv3:starting", gic_starting_cpu, NULL); + + /* Register all 8 non-secure SGIs */ + base_sgi = __irq_domain_alloc_irqs(gic_data.domain, -1, 8, + NUMA_NO_NODE, &sgi_fwspec, + false, NULL); + if (WARN_ON(base_sgi <= 0)) + return; + + set_smp_ipi_range(base_sgi, 8); } static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, @@ -1215,6 +1210,7 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, } #else #define gic_set_affinity NULL +#define gic_ipi_send_mask NULL #define gic_smp_init() do { } while(0) #endif @@ -1257,6 +1253,7 @@ static struct irq_chip gic_chip = { .irq_set_irqchip_state = gic_irq_set_irqchip_state, .irq_nmi_setup = gic_irq_nmi_setup, .irq_nmi_teardown = gic_irq_nmi_teardown, + .ipi_send_mask = gic_ipi_send_mask, .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, @@ -1274,6 +1271,7 @@ static struct irq_chip gic_eoimode1_chip = { .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, .irq_nmi_setup = gic_irq_nmi_setup, .irq_nmi_teardown = gic_irq_nmi_teardown, + .ipi_send_mask = gic_ipi_send_mask, .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, @@ -1289,6 +1287,12 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, switch (__get_intid_range(hw)) { case SGI_RANGE: + irq_set_percpu_devid(irq); + irq_domain_set_info(d, irq, hw, chip, d->host_data, + handle_percpu_devid_fasteoi_ipi, + NULL, NULL); + break; + case PPI_RANGE: case EPPI_RANGE: irq_set_percpu_devid(irq); @@ -1318,13 +1322,17 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, return 0; } -#define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) - static int gic_irq_domain_translate(struct irq_domain *d, struct irq_fwspec *fwspec, unsigned long *hwirq, unsigned int *type) { + if (fwspec->param_count == 1 && fwspec->param[0] < 16) { + *hwirq = fwspec->param[0]; + *type = IRQ_TYPE_EDGE_RISING; + return 0; + } + if (is_of_node(fwspec->fwnode)) { if (fwspec->param_count < 3) return -EINVAL; @@ -1656,9 +1664,9 @@ static int __init gic_init_bases(void __iomem *dist_base, gic_update_rdist_properties(); - gic_smp_init(); gic_dist_init(); gic_cpu_init(); + gic_smp_init(); gic_cpu_pm_init(); if (gic_dist_supports_lpis()) { From 7ec46b519467852fc8eb83b6214ad568f8007846 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 25 Apr 2020 15:24:01 +0100 Subject: [PATCH 07/18] irqchip/gic: Refactor SMP configuration As we are about to change quite a lot of the SMP support code, let's start by moving it around so that it minimizes the amount of #ifdefery. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-gic.c | 76 ++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index a27ba2cc1dce..4ffd62af888f 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -325,28 +325,6 @@ static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) return 0; } -#ifdef CONFIG_SMP -static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, - bool force) -{ - void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + gic_irq(d); - unsigned int cpu; - - if (!force) - cpu = cpumask_any_and(mask_val, cpu_online_mask); - else - cpu = cpumask_first(mask_val); - - if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids) - return -EINVAL; - - writeb_relaxed(gic_cpu_map[cpu], reg); - irq_data_update_effective_affinity(d, cpumask_of(cpu)); - - return IRQ_SET_MASK_OK_DONE; -} -#endif - static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) { u32 irqstat, irqnr; @@ -795,6 +773,26 @@ static int gic_pm_init(struct gic_chip_data *gic) #endif #ifdef CONFIG_SMP +static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, + bool force) +{ + void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + gic_irq(d); + unsigned int cpu; + + if (!force) + cpu = cpumask_any_and(mask_val, cpu_online_mask); + else + cpu = cpumask_first(mask_val); + + if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids) + return -EINVAL; + + writeb_relaxed(gic_cpu_map[cpu], reg); + irq_data_update_effective_affinity(d, cpumask_of(cpu)); + + return IRQ_SET_MASK_OK_DONE; +} + static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) { int cpu; @@ -824,6 +822,23 @@ static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) gic_unlock_irqrestore(flags); } + +static int gic_starting_cpu(unsigned int cpu) +{ + gic_cpu_init(&gic_data[0]); + return 0; +} + +static __init void gic_smp_init(void) +{ + set_smp_cross_call(gic_raise_softirq); + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, + "irqchip/arm/gic:starting", + gic_starting_cpu, NULL); +} +#else +#define gic_smp_init() do { } while(0) +#define gic_set_affinity NULL #endif #ifdef CONFIG_BL_SWITCHER @@ -1027,12 +1042,6 @@ static int gic_irq_domain_translate(struct irq_domain *d, return -EINVAL; } -static int gic_starting_cpu(unsigned int cpu) -{ - gic_cpu_init(&gic_data[0]); - return 0; -} - static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *arg) { @@ -1079,10 +1088,8 @@ static void gic_init_chip(struct gic_chip_data *gic, struct device *dev, gic->chip.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity; } -#ifdef CONFIG_SMP if (gic == &gic_data[0]) gic->chip.irq_set_affinity = gic_set_affinity; -#endif } static int gic_init_bases(struct gic_chip_data *gic, @@ -1199,12 +1206,7 @@ static int __init __gic_init_bases(struct gic_chip_data *gic, */ for (i = 0; i < NR_GIC_CPU_IF; i++) gic_cpu_map[i] = 0xff; -#ifdef CONFIG_SMP - set_smp_cross_call(gic_raise_softirq); -#endif - cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, - "irqchip/arm/gic:starting", - gic_starting_cpu, NULL); + set_handle_irq(gic_handle_irq); if (static_branch_likely(&supports_deactivate_key)) pr_info("GIC: Using split EOI/Deactivate mode\n"); @@ -1221,6 +1223,8 @@ static int __init __gic_init_bases(struct gic_chip_data *gic, ret = gic_init_bases(gic, handle); if (ret) kfree(name); + else if (gic == &gic_data[0]) + gic_smp_init(); return ret; } From 64a267e9a41c5a91efdfa5bf55bd2509cb4f7170 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 25 Apr 2020 15:24:01 +0100 Subject: [PATCH 08/18] irqchip/gic: Configure SGIs as standard interrupts Change the way we deal with GIC SGIs by turning them into proper IRQs, and calling into the arch code to register the interrupt range instead of a callback. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-gic.c | 130 ++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 42 deletions(-) diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 4ffd62af888f..66671e1a6106 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -124,6 +124,8 @@ static struct gic_chip_data gic_data[CONFIG_ARM_GIC_MAX_NR] __read_mostly; static struct gic_kvm_info gic_v2_kvm_info; +static DEFINE_PER_CPU(u32, sgi_intid); + #ifdef CONFIG_GIC_NON_BANKED static void __iomem *gic_get_percpu_base(union gic_base *base) { @@ -226,16 +228,26 @@ static void gic_unmask_irq(struct irq_data *d) static void gic_eoi_irq(struct irq_data *d) { - writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI); + u32 hwirq = gic_irq(d); + + if (hwirq < 16) + hwirq = this_cpu_read(sgi_intid); + + writel_relaxed(hwirq, gic_cpu_base(d) + GIC_CPU_EOI); } static void gic_eoimode1_eoi_irq(struct irq_data *d) { + u32 hwirq = gic_irq(d); + /* Do not deactivate an IRQ forwarded to a vcpu. */ if (irqd_is_forwarded_to_vcpu(d)) return; - writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_DEACTIVATE); + if (hwirq < 16) + hwirq = this_cpu_read(sgi_intid); + + writel_relaxed(hwirq, gic_cpu_base(d) + GIC_CPU_DEACTIVATE); } static int gic_irq_set_irqchip_state(struct irq_data *d, @@ -295,7 +307,7 @@ static int gic_set_type(struct irq_data *d, unsigned int type) /* Interrupt configuration for SGIs can't be changed */ if (gicirq < 16) - return -EINVAL; + return type == IRQ_TYPE_EDGE_RISING ? 0 : -EINVAL; /* SPIs have restrictions on the supported types */ if (gicirq >= 32 && type != IRQ_TYPE_LEVEL_HIGH && @@ -315,7 +327,7 @@ static int gic_set_type(struct irq_data *d, unsigned int type) static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) { /* Only interrupts on the primary GIC can be forwarded to a vcpu. */ - if (cascading_gic_irq(d)) + if (cascading_gic_irq(d) || gic_irq(d) < 16) return -EINVAL; if (vcpu) @@ -335,31 +347,33 @@ static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK); irqnr = irqstat & GICC_IAR_INT_ID_MASK; - if (likely(irqnr > 15 && irqnr < 1020)) { - if (static_branch_likely(&supports_deactivate_key)) - writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI); - isb(); - handle_domain_irq(gic->domain, irqnr, regs); - continue; - } - if (irqnr < 16) { + if (unlikely(irqnr >= 1020)) + break; + + if (static_branch_likely(&supports_deactivate_key)) writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI); - if (static_branch_likely(&supports_deactivate_key)) - writel_relaxed(irqstat, cpu_base + GIC_CPU_DEACTIVATE); -#ifdef CONFIG_SMP - /* - * Ensure any shared data written by the CPU sending - * the IPI is read after we've read the ACK register - * on the GIC. - * - * Pairs with the write barrier in gic_raise_softirq - */ + isb(); + + /* + * Ensure any shared data written by the CPU sending the IPI + * is read after we've read the ACK register on the GIC. + * + * Pairs with the write barrier in gic_ipi_send_mask + */ + if (irqnr <= 15) { smp_rmb(); - handle_IPI(irqnr, regs); -#endif - continue; + + /* + * The GIC encodes the source CPU in GICC_IAR, + * leading to the deactivation to fail if not + * written back as is to GICC_EOI. Stash the INTID + * away for gic_eoi_irq() to write back. This only + * works because we don't nest SGIs... + */ + this_cpu_write(sgi_intid, irqstat); } - break; + + handle_domain_irq(gic->domain, irqnr, regs); } while (1); } @@ -793,14 +807,14 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, return IRQ_SET_MASK_OK_DONE; } -static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) +static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { int cpu; unsigned long flags, map = 0; if (unlikely(nr_cpu_ids == 1)) { /* Only one CPU? let's do a self-IPI... */ - writel_relaxed(2 << 24 | irq, + writel_relaxed(2 << 24 | d->hwirq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT); return; } @@ -818,7 +832,7 @@ static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) dmb(ishst); /* this always happens on GIC0 */ - writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT); + writel_relaxed(map << 16 | d->hwirq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT); gic_unlock_irqrestore(flags); } @@ -831,14 +845,28 @@ static int gic_starting_cpu(unsigned int cpu) static __init void gic_smp_init(void) { - set_smp_cross_call(gic_raise_softirq); + struct irq_fwspec sgi_fwspec = { + .fwnode = gic_data[0].domain->fwnode, + .param_count = 1, + }; + int base_sgi; + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, "irqchip/arm/gic:starting", gic_starting_cpu, NULL); + + base_sgi = __irq_domain_alloc_irqs(gic_data[0].domain, -1, 8, + NUMA_NO_NODE, &sgi_fwspec, + false, NULL); + if (WARN_ON(base_sgi <= 0)) + return; + + set_smp_ipi_range(base_sgi, 8); } #else #define gic_smp_init() do { } while(0) #define gic_set_affinity NULL +#define gic_ipi_send_mask NULL #endif #ifdef CONFIG_BL_SWITCHER @@ -985,15 +1013,24 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, { struct gic_chip_data *gic = d->host_data; - if (hw < 32) { + switch (hw) { + case 0 ... 15: + irq_set_percpu_devid(irq); + irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data, + handle_percpu_devid_fasteoi_ipi, + NULL, NULL); + break; + case 16 ... 31: irq_set_percpu_devid(irq); irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data, handle_percpu_devid_irq, NULL, NULL); - } else { + break; + default: irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data, handle_fasteoi_irq, NULL, NULL); irq_set_probe(irq); irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); + break; } return 0; } @@ -1007,19 +1044,26 @@ static int gic_irq_domain_translate(struct irq_domain *d, unsigned long *hwirq, unsigned int *type) { + if (fwspec->param_count == 1 && fwspec->param[0] < 16) { + *hwirq = fwspec->param[0]; + *type = IRQ_TYPE_EDGE_RISING; + return 0; + } + if (is_of_node(fwspec->fwnode)) { if (fwspec->param_count < 3) return -EINVAL; - /* Get the interrupt number and add 16 to skip over SGIs */ - *hwirq = fwspec->param[1] + 16; - - /* - * For SPIs, we need to add 16 more to get the GIC irq - * ID number - */ - if (!fwspec->param[0]) - *hwirq += 16; + switch (fwspec->param[0]) { + case 0: /* SPI */ + *hwirq = fwspec->param[1] + 32; + break; + case 1: /* PPI */ + *hwirq = fwspec->param[1] + 16; + break; + default: + return -EINVAL; + } *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; @@ -1088,8 +1132,10 @@ static void gic_init_chip(struct gic_chip_data *gic, struct device *dev, gic->chip.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity; } - if (gic == &gic_data[0]) + if (gic == &gic_data[0]) { gic->chip.irq_set_affinity = gic_set_affinity; + gic->chip.ipi_send_mask = gic_ipi_send_mask; + } } static int gic_init_bases(struct gic_chip_data *gic, From 3567c6ca47546106d36d995790e4eb80e3f14632 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 May 2020 09:42:46 +0100 Subject: [PATCH 09/18] irqchip/gic-common: Don't enable SGIs by default The architecture code now enables the IPIs as required, so no need to enable SGIs by default in the GIC code. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-gic-common.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/irqchip/irq-gic-common.c b/drivers/irqchip/irq-gic-common.c index 82520006195d..f47b41dfd023 100644 --- a/drivers/irqchip/irq-gic-common.c +++ b/drivers/irqchip/irq-gic-common.c @@ -152,9 +152,6 @@ void gic_cpu_config(void __iomem *base, int nr, void (*sync_access)(void)) writel_relaxed(GICD_INT_DEF_PRI_X4, base + GIC_DIST_PRI + i * 4 / 4); - /* Ensure all SGI interrupts are now enabled */ - writel_relaxed(GICD_INT_EN_SET_SGI, base + GIC_DIST_ENABLE_SET); - if (sync_access) sync_access(); } From 0809ae724904c3c5dbdddf4169d48aac9c6fcdc8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 5 May 2020 12:59:04 +0100 Subject: [PATCH 10/18] irqchip/bcm2836: Configure mailbox interrupts as standard interrupts In order to switch the bcm2836 driver to privide standard interrupts for IPIs, it first needs to stop lying about the way things work. The mailbox interrupt is actually a multiplexer, with enough bits to store 32 pending interrupts per CPU. So let's turn it into a chained irqchip. Once this is done, we can instanciate the corresponding IPIs, and pass them to the architecture code. Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-bcm2836.c | 157 +++++++++++++++++++++++++++------- 1 file changed, 128 insertions(+), 29 deletions(-) diff --git a/drivers/irqchip/irq-bcm2836.c b/drivers/irqchip/irq-bcm2836.c index 2038693f074c..85df6ddad9be 100644 --- a/drivers/irqchip/irq-bcm2836.c +++ b/drivers/irqchip/irq-bcm2836.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -89,12 +90,24 @@ static struct irq_chip bcm2836_arm_irqchip_gpu = { .irq_unmask = bcm2836_arm_irqchip_unmask_gpu_irq, }; +static void bcm2836_arm_irqchip_dummy_op(struct irq_data *d) +{ +} + +static struct irq_chip bcm2836_arm_irqchip_dummy = { + .name = "bcm2836-dummy", + .irq_eoi = bcm2836_arm_irqchip_dummy_op, +}; + static int bcm2836_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) { struct irq_chip *chip; switch (hw) { + case LOCAL_IRQ_MAILBOX0: + chip = &bcm2836_arm_irqchip_dummy; + break; case LOCAL_IRQ_CNTPSIRQ: case LOCAL_IRQ_CNTPNSIRQ: case LOCAL_IRQ_CNTHPIRQ: @@ -127,17 +140,7 @@ __exception_irq_entry bcm2836_arm_irqchip_handle_irq(struct pt_regs *regs) u32 stat; stat = readl_relaxed(intc.base + LOCAL_IRQ_PENDING0 + 4 * cpu); - if (stat & BIT(LOCAL_IRQ_MAILBOX0)) { -#ifdef CONFIG_SMP - void __iomem *mailbox0 = (intc.base + - LOCAL_MAILBOX0_CLR0 + 16 * cpu); - u32 mbox_val = readl(mailbox0); - u32 ipi = ffs(mbox_val) - 1; - - writel(1 << ipi, mailbox0); - handle_IPI(ipi, regs); -#endif - } else if (stat) { + if (stat) { u32 hwirq = ffs(stat) - 1; handle_domain_irq(intc.domain, hwirq, regs); @@ -145,8 +148,35 @@ __exception_irq_entry bcm2836_arm_irqchip_handle_irq(struct pt_regs *regs) } #ifdef CONFIG_SMP -static void bcm2836_arm_irqchip_send_ipi(const struct cpumask *mask, - unsigned int ipi) +static struct irq_domain *ipi_domain; + +static void bcm2836_arm_irqchip_handle_ipi(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + int cpu = smp_processor_id(); + u32 mbox_val; + + chained_irq_enter(chip, desc); + + mbox_val = readl_relaxed(intc.base + LOCAL_MAILBOX0_CLR0 + 16 * cpu); + if (mbox_val) { + int hwirq = ffs(mbox_val) - 1; + generic_handle_irq(irq_find_mapping(ipi_domain, hwirq)); + } + + chained_irq_exit(chip, desc); +} + +static void bcm2836_arm_irqchip_ipi_eoi(struct irq_data *d) +{ + int cpu = smp_processor_id(); + + writel_relaxed(BIT(d->hwirq), + intc.base + LOCAL_MAILBOX0_CLR0 + 16 * cpu); +} + +static void bcm2836_arm_irqchip_ipi_send_mask(struct irq_data *d, + const struct cpumask *mask) { int cpu; void __iomem *mailbox0_base = intc.base + LOCAL_MAILBOX0_SET0; @@ -157,11 +187,45 @@ static void bcm2836_arm_irqchip_send_ipi(const struct cpumask *mask, */ smp_wmb(); - for_each_cpu(cpu, mask) { - writel(1 << ipi, mailbox0_base + 16 * cpu); - } + for_each_cpu(cpu, mask) + writel_relaxed(BIT(d->hwirq), mailbox0_base + 16 * cpu); } +static struct irq_chip bcm2836_arm_irqchip_ipi = { + .name = "IPI", + .irq_eoi = bcm2836_arm_irqchip_ipi_eoi, + .ipi_send_mask = bcm2836_arm_irqchip_ipi_send_mask, +}; + +static int bcm2836_arm_irqchip_ipi_alloc(struct irq_domain *d, + unsigned int virq, + unsigned int nr_irqs, void *args) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(d, virq + i, i, &bcm2836_arm_irqchip_ipi, + d->host_data, + handle_percpu_devid_fasteoi_ipi, + NULL, NULL); + } + + return 0; +} + +static void bcm2836_arm_irqchip_ipi_free(struct irq_domain *d, + unsigned int virq, + unsigned int nr_irqs) +{ + /* Not freeing IPIs */ +} + +static const struct irq_domain_ops ipi_domain_ops = { + .alloc = bcm2836_arm_irqchip_ipi_alloc, + .free = bcm2836_arm_irqchip_ipi_free, +}; + static int bcm2836_cpu_starting(unsigned int cpu) { bcm2836_arm_irqchip_unmask_per_cpu_irq(LOCAL_MAILBOX_INT_CONTROL0, 0, @@ -175,6 +239,52 @@ static int bcm2836_cpu_dying(unsigned int cpu) cpu); return 0; } + +#define BITS_PER_MBOX 32 + +static void bcm2836_arm_irqchip_smp_init(void) +{ + struct irq_fwspec ipi_fwspec = { + .fwnode = intc.domain->fwnode, + .param_count = 1, + .param = { + [0] = LOCAL_IRQ_MAILBOX0, + }, + }; + int base_ipi, mux_irq; + + mux_irq = irq_create_fwspec_mapping(&ipi_fwspec); + if (WARN_ON(mux_irq <= 0)) + return; + + ipi_domain = irq_domain_create_linear(intc.domain->fwnode, + BITS_PER_MBOX, &ipi_domain_ops, + NULL); + if (WARN_ON(!ipi_domain)) + return; + + ipi_domain->flags |= IRQ_DOMAIN_FLAG_IPI_SINGLE; + irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); + + base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, BITS_PER_MBOX, + NUMA_NO_NODE, NULL, + false, NULL); + + if (WARN_ON(!base_ipi)) + return; + + set_smp_ipi_range(base_ipi, BITS_PER_MBOX); + + irq_set_chained_handler_and_data(mux_irq, + bcm2836_arm_irqchip_handle_ipi, NULL); + + /* Unmask IPIs to the boot CPU. */ + cpuhp_setup_state(CPUHP_AP_IRQ_BCM2836_STARTING, + "irqchip/bcm2836:starting", bcm2836_cpu_starting, + bcm2836_cpu_dying); +} +#else +#define bcm2836_arm_irqchip_smp_init() do { } while(0) #endif static const struct irq_domain_ops bcm2836_arm_irqchip_intc_ops = { @@ -182,19 +292,6 @@ static const struct irq_domain_ops bcm2836_arm_irqchip_intc_ops = { .map = bcm2836_map, }; -static void -bcm2836_arm_irqchip_smp_init(void) -{ -#ifdef CONFIG_SMP - /* Unmask IPIs to the boot CPU. */ - cpuhp_setup_state(CPUHP_AP_IRQ_BCM2836_STARTING, - "irqchip/bcm2836:starting", bcm2836_cpu_starting, - bcm2836_cpu_dying); - - set_smp_cross_call(bcm2836_arm_irqchip_send_ipi); -#endif -} - /* * The LOCAL_IRQ_CNT* timer firings are based off of the external * oscillator with some scaling. The firmware sets up CNTFRQ to @@ -232,6 +329,8 @@ static int __init bcm2836_arm_irqchip_l1_intc_of_init(struct device_node *node, if (!intc.domain) panic("%pOF: unable to create IRQ domain\n", node); + irq_domain_update_bus_token(intc.domain, DOMAIN_BUS_WIRED); + bcm2836_arm_irqchip_smp_init(); set_handle_irq(bcm2836_arm_irqchip_handle_irq); From a2df12c5899e9bb181cb64385b04f2bc755780b6 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 20 Jun 2020 20:02:18 +0100 Subject: [PATCH 11/18] irqchip/hip04: Configure IPIs as standard interrupts In order to switch the hip04 driver to provide standard interrupts for IPIs, rework the way interrupts are allocated, making sure the irqdomain covers the SGIs as well as the rest of the interrupt range. The driver is otherwise so old-school that it creates all interrupts upfront (duh!), so there is hardly anything else to change, apart from communicating the IPIs to the arch code. Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-hip04.c | 89 +++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 49 deletions(-) diff --git a/drivers/irqchip/irq-hip04.c b/drivers/irqchip/irq-hip04.c index 130caa1c9d93..9b73dcfaf48d 100644 --- a/drivers/irqchip/irq-hip04.c +++ b/drivers/irqchip/irq-hip04.c @@ -171,6 +171,29 @@ static int hip04_irq_set_affinity(struct irq_data *d, return IRQ_SET_MASK_OK; } + +static void hip04_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) +{ + int cpu; + unsigned long flags, map = 0; + + raw_spin_lock_irqsave(&irq_controller_lock, flags); + + /* Convert our logical CPU mask into a physical one. */ + for_each_cpu(cpu, mask) + map |= hip04_cpu_map[cpu]; + + /* + * Ensure that stores to Normal memory are visible to the + * other CPUs before they observe us issuing the IPI. + */ + dmb(ishst); + + /* this always happens on GIC0 */ + writel_relaxed(map << 8 | d->hwirq, hip04_data.dist_base + GIC_DIST_SOFTINT); + + raw_spin_unlock_irqrestore(&irq_controller_lock, flags); +} #endif static void __exception_irq_entry hip04_handle_irq(struct pt_regs *regs) @@ -182,19 +205,9 @@ static void __exception_irq_entry hip04_handle_irq(struct pt_regs *regs) irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK); irqnr = irqstat & GICC_IAR_INT_ID_MASK; - if (likely(irqnr > 15 && irqnr <= HIP04_MAX_IRQS)) { + if (irqnr <= HIP04_MAX_IRQS) handle_domain_irq(hip04_data.domain, irqnr, regs); - continue; - } - if (irqnr < 16) { - writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI); -#ifdef CONFIG_SMP - handle_IPI(irqnr, regs); -#endif - continue; - } - break; - } while (1); + } while (irqnr > HIP04_MAX_IRQS); } static struct irq_chip hip04_irq_chip = { @@ -205,6 +218,7 @@ static struct irq_chip hip04_irq_chip = { .irq_set_type = hip04_irq_set_type, #ifdef CONFIG_SMP .irq_set_affinity = hip04_irq_set_affinity, + .ipi_send_mask = hip04_ipi_send_mask, #endif .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | @@ -279,39 +293,17 @@ static void hip04_irq_cpu_init(struct hip04_irq_data *intc) writel_relaxed(1, base + GIC_CPU_CTRL); } -#ifdef CONFIG_SMP -static void hip04_raise_softirq(const struct cpumask *mask, unsigned int irq) -{ - int cpu; - unsigned long flags, map = 0; - - raw_spin_lock_irqsave(&irq_controller_lock, flags); - - /* Convert our logical CPU mask into a physical one. */ - for_each_cpu(cpu, mask) - map |= hip04_cpu_map[cpu]; - - /* - * Ensure that stores to Normal memory are visible to the - * other CPUs before they observe us issuing the IPI. - */ - dmb(ishst); - - /* this always happens on GIC0 */ - writel_relaxed(map << 8 | irq, hip04_data.dist_base + GIC_DIST_SOFTINT); - - raw_spin_unlock_irqrestore(&irq_controller_lock, flags); -} -#endif - static int hip04_irq_domain_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) { - if (hw < 32) { + if (hw < 16) { + irq_set_percpu_devid(irq); + irq_set_chip_and_handler(irq, &hip04_irq_chip, + handle_percpu_devid_fasteoi_ipi); + } else if (hw < 32) { irq_set_percpu_devid(irq); irq_set_chip_and_handler(irq, &hip04_irq_chip, handle_percpu_devid_irq); - irq_set_status_flags(irq, IRQ_NOAUTOEN); } else { irq_set_chip_and_handler(irq, &hip04_irq_chip, handle_fasteoi_irq); @@ -328,10 +320,13 @@ static int hip04_irq_domain_xlate(struct irq_domain *d, unsigned long *out_hwirq, unsigned int *out_type) { - unsigned long ret = 0; - if (irq_domain_get_of_node(d) != controller) return -EINVAL; + if (intsize == 1 && intspec[0] < 16) { + *out_hwirq = intspec[0]; + *out_type = IRQ_TYPE_EDGE_RISING; + return 0; + } if (intsize < 3) return -EINVAL; @@ -344,7 +339,7 @@ static int hip04_irq_domain_xlate(struct irq_domain *d, *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK; - return ret; + return 0; } static int hip04_irq_starting_cpu(unsigned int cpu) @@ -361,7 +356,6 @@ static const struct irq_domain_ops hip04_irq_domain_ops = { static int __init hip04_of_init(struct device_node *node, struct device_node *parent) { - irq_hw_number_t hwirq_base = 16; int nr_irqs, irq_base, i; if (WARN_ON(!node)) @@ -390,24 +384,21 @@ hip04_of_init(struct device_node *node, struct device_node *parent) nr_irqs = HIP04_MAX_IRQS; hip04_data.nr_irqs = nr_irqs; - nr_irqs -= hwirq_base; /* calculate # of irqs to allocate */ - - irq_base = irq_alloc_descs(-1, hwirq_base, nr_irqs, numa_node_id()); + irq_base = irq_alloc_descs(-1, 0, nr_irqs, numa_node_id()); if (irq_base < 0) { pr_err("failed to allocate IRQ numbers\n"); return -EINVAL; } hip04_data.domain = irq_domain_add_legacy(node, nr_irqs, irq_base, - hwirq_base, + 0, &hip04_irq_domain_ops, &hip04_data); - if (WARN_ON(!hip04_data.domain)) return -EINVAL; #ifdef CONFIG_SMP - set_smp_cross_call(hip04_raise_softirq); + set_smp_ipi_range(irq_base, 16); #endif set_handle_irq(hip04_handle_irq); From f02147dd02eb5fab31b55b73e7524f94b5f20324 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 22 Jun 2020 21:23:36 +0100 Subject: [PATCH 12/18] irqchip/armada-370-xp: Configure IPIs as standard interrupts To introduce IPIs as standard interrupts to the Armada 370-XP driver, let's allocate a completely separate irqdomain and irqchip combo that lives parallel to the "standard" one. This effectively should be modelled as a chained interrupt controller, but the code is in such a state that it is pretty hard to shoehorn, as it would require the rewrite of the MSI layer as well. Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-armada-370-xp.c | 302 ++++++++++++++++++---------- 1 file changed, 198 insertions(+), 104 deletions(-) diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index c9bdc5221b82..d7eb2e93db8f 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -310,7 +310,134 @@ static inline int armada_370_xp_msi_init(struct device_node *node, } #endif +static void armada_xp_mpic_perf_init(void) +{ + unsigned long cpuid = cpu_logical_map(smp_processor_id()); + + /* Enable Performance Counter Overflow interrupts */ + writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid), + per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS); +} + #ifdef CONFIG_SMP +static struct irq_domain *ipi_domain; + +static void armada_370_xp_ipi_mask(struct irq_data *d) +{ + u32 reg; + reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + reg &= ~BIT(d->hwirq); + writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); +} + +static void armada_370_xp_ipi_unmask(struct irq_data *d) +{ + u32 reg; + reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + reg |= BIT(d->hwirq); + writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); +} + +static void armada_370_xp_ipi_send_mask(struct irq_data *d, + const struct cpumask *mask) +{ + unsigned long map = 0; + int cpu; + + /* Convert our logical CPU mask into a physical one. */ + for_each_cpu(cpu, mask) + map |= 1 << cpu_logical_map(cpu); + + /* + * Ensure that stores to Normal memory are visible to the + * other CPUs before issuing the IPI. + */ + dsb(); + + /* submit softirq */ + writel((map << 8) | d->hwirq, main_int_base + + ARMADA_370_XP_SW_TRIG_INT_OFFS); +} + +static void armada_370_xp_ipi_eoi(struct irq_data *d) +{ + writel(~BIT(d->hwirq), per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); +} + +static struct irq_chip ipi_irqchip = { + .name = "IPI", + .irq_mask = armada_370_xp_ipi_mask, + .irq_unmask = armada_370_xp_ipi_unmask, + .irq_eoi = armada_370_xp_ipi_eoi, + .ipi_send_mask = armada_370_xp_ipi_send_mask, +}; + +static int armada_370_xp_ipi_alloc(struct irq_domain *d, + unsigned int virq, + unsigned int nr_irqs, void *args) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_set_percpu_devid(virq + i); + irq_domain_set_info(d, virq + i, i, &ipi_irqchip, + d->host_data, + handle_percpu_devid_fasteoi_ipi, + NULL, NULL); + } + + return 0; +} + +static void armada_370_xp_ipi_free(struct irq_domain *d, + unsigned int virq, + unsigned int nr_irqs) +{ + /* Not freeing IPIs */ +} + +static const struct irq_domain_ops ipi_domain_ops = { + .alloc = armada_370_xp_ipi_alloc, + .free = armada_370_xp_ipi_free, +}; + +static void ipi_resume(void) +{ + int i; + + for (i = 0; i < IPI_DOORBELL_END; i++) { + int irq; + + irq = irq_find_mapping(ipi_domain, i); + if (irq <= 0) + continue; + if (irq_percpu_is_enabled(irq)) { + struct irq_data *d; + d = irq_domain_get_irq_data(ipi_domain, irq); + armada_370_xp_ipi_unmask(d); + } + } +} + +static __init void armada_xp_ipi_init(struct device_node *node) +{ + int base_ipi; + + ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), + IPI_DOORBELL_END, + &ipi_domain_ops, NULL); + if (WARN_ON(!ipi_domain)) + return; + + irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); + base_ipi = __irq_domain_alloc_irqs(ipi_domain, -1, IPI_DOORBELL_END, + NUMA_NO_NODE, NULL, false, NULL); + if (WARN_ON(!base_ipi)) + return; + + set_smp_ipi_range(base_ipi, IPI_DOORBELL_END); +} + static DEFINE_RAW_SPINLOCK(irq_controller_lock); static int armada_xp_set_affinity(struct irq_data *d, @@ -334,6 +461,70 @@ static int armada_xp_set_affinity(struct irq_data *d, return IRQ_SET_MASK_OK; } + +static void armada_xp_mpic_smp_cpu_init(void) +{ + u32 control; + int nr_irqs, i; + + control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); + nr_irqs = (control >> 2) & 0x3ff; + + for (i = 0; i < nr_irqs; i++) + writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS); + + /* Disable all IPIs */ + writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + + /* Clear pending IPIs */ + writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); + + /* Unmask IPI interrupt */ + writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); +} + +static void armada_xp_mpic_reenable_percpu(void) +{ + unsigned int irq; + + /* Re-enable per-CPU interrupts that were enabled before suspend */ + for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) { + struct irq_data *data; + int virq; + + virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); + if (virq == 0) + continue; + + data = irq_get_irq_data(virq); + + if (!irq_percpu_is_enabled(virq)) + continue; + + armada_370_xp_irq_unmask(data); + } + + ipi_resume(); +} + +static int armada_xp_mpic_starting_cpu(unsigned int cpu) +{ + armada_xp_mpic_perf_init(); + armada_xp_mpic_smp_cpu_init(); + armada_xp_mpic_reenable_percpu(); + return 0; +} + +static int mpic_cascaded_starting_cpu(unsigned int cpu) +{ + armada_xp_mpic_perf_init(); + armada_xp_mpic_reenable_percpu(); + enable_percpu_irq(parent_irq, IRQ_TYPE_NONE); + return 0; +} +#else +static void armada_xp_mpic_smp_cpu_init(void) {} +static void ipi_resume(void) {} #endif static struct irq_chip armada_370_xp_irq_chip = { @@ -372,98 +563,6 @@ static int armada_370_xp_mpic_irq_map(struct irq_domain *h, return 0; } -static void armada_xp_mpic_smp_cpu_init(void) -{ - u32 control; - int nr_irqs, i; - - control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); - nr_irqs = (control >> 2) & 0x3ff; - - for (i = 0; i < nr_irqs; i++) - writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS); - - /* Clear pending IPIs */ - writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); - - /* Enable first 8 IPIs */ - writel(IPI_DOORBELL_MASK, per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_MSK_OFFS); - - /* Unmask IPI interrupt */ - writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); -} - -static void armada_xp_mpic_perf_init(void) -{ - unsigned long cpuid = cpu_logical_map(smp_processor_id()); - - /* Enable Performance Counter Overflow interrupts */ - writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid), - per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS); -} - -#ifdef CONFIG_SMP -static void armada_mpic_send_doorbell(const struct cpumask *mask, - unsigned int irq) -{ - int cpu; - unsigned long map = 0; - - /* Convert our logical CPU mask into a physical one. */ - for_each_cpu(cpu, mask) - map |= 1 << cpu_logical_map(cpu); - - /* - * Ensure that stores to Normal memory are visible to the - * other CPUs before issuing the IPI. - */ - dsb(); - - /* submit softirq */ - writel((map << 8) | irq, main_int_base + - ARMADA_370_XP_SW_TRIG_INT_OFFS); -} - -static void armada_xp_mpic_reenable_percpu(void) -{ - unsigned int irq; - - /* Re-enable per-CPU interrupts that were enabled before suspend */ - for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) { - struct irq_data *data; - int virq; - - virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); - if (virq == 0) - continue; - - data = irq_get_irq_data(virq); - - if (!irq_percpu_is_enabled(virq)) - continue; - - armada_370_xp_irq_unmask(data); - } -} - -static int armada_xp_mpic_starting_cpu(unsigned int cpu) -{ - armada_xp_mpic_perf_init(); - armada_xp_mpic_smp_cpu_init(); - armada_xp_mpic_reenable_percpu(); - return 0; -} - -static int mpic_cascaded_starting_cpu(unsigned int cpu) -{ - armada_xp_mpic_perf_init(); - armada_xp_mpic_reenable_percpu(); - enable_percpu_irq(parent_irq, IRQ_TYPE_NONE); - return 0; -} -#endif - static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = { .map = armada_370_xp_mpic_irq_map, .xlate = irq_domain_xlate_onecell, @@ -562,22 +661,15 @@ armada_370_xp_handle_irq(struct pt_regs *regs) #ifdef CONFIG_SMP /* IPI Handling */ if (irqnr == 0) { - u32 ipimask, ipinr; + unsigned long ipimask; + int ipi; ipimask = readl_relaxed(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) & IPI_DOORBELL_MASK; - writel(~ipimask, per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); - - /* Handle all pending doorbells */ - for (ipinr = IPI_DOORBELL_START; - ipinr < IPI_DOORBELL_END; ipinr++) { - if (ipimask & (0x1 << ipinr)) - handle_IPI(ipinr, regs); - } - continue; + for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END) + handle_domain_irq(ipi_domain, ipi, regs); } #endif @@ -636,6 +728,8 @@ static void armada_370_xp_mpic_resume(void) writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK) writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + + ipi_resume(); } static struct syscore_ops armada_370_xp_mpic_syscore_ops = { @@ -691,7 +785,7 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, irq_set_default_host(armada_370_xp_mpic_domain); set_handle_irq(armada_370_xp_handle_irq); #ifdef CONFIG_SMP - set_smp_cross_call(armada_mpic_send_doorbell); + armada_xp_ipi_init(node); cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING, "irqchip/armada/ipi:starting", armada_xp_mpic_starting_cpu, NULL); From 5cebfd2d47c214f69d918e3d34ad183c061eddb2 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 9 May 2020 14:00:23 +0100 Subject: [PATCH 13/18] arm64: Kill __smp_cross_call and co The old IPI registration interface is now unused on arm64, so let's get rid of it. Reviewed-by: Valentin Schneider Acked-by: Catalin Marinas Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/irq_work.h | 4 +--- arch/arm64/include/asm/smp.h | 12 ---------- arch/arm64/kernel/smp.c | 38 ++++++------------------------- 3 files changed, 8 insertions(+), 46 deletions(-) diff --git a/arch/arm64/include/asm/irq_work.h b/arch/arm64/include/asm/irq_work.h index 8a1ef1907760..a1020285ea75 100644 --- a/arch/arm64/include/asm/irq_work.h +++ b/arch/arm64/include/asm/irq_work.h @@ -2,11 +2,9 @@ #ifndef __ASM_IRQ_WORK_H #define __ASM_IRQ_WORK_H -#include - static inline bool arch_irq_work_has_interrupt(void) { - return !!__smp_cross_call; + return true; } #endif /* __ASM_IRQ_WORK_H */ diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 57c5db15f6b7..c298ad02252e 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -60,24 +60,12 @@ struct seq_file; */ extern void show_ipi_list(struct seq_file *p, int prec); -/* - * Called from C code, this handles an IPI. - */ -extern void handle_IPI(int ipinr, struct pt_regs *regs); - /* * Discover the set of possible CPUs and determine their * SMP operations. */ extern void smp_init_cpus(void); -/* - * Provide a function to raise an IPI cross call on CPUs in callmap. - */ -extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); - -extern void (*__smp_cross_call)(const struct cpumask *, unsigned int); - /* * Register IPI interrupts with the arch SMP code */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 00c9db1b61b5..58fb155fb0ab 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -782,13 +782,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) } } -void (*__smp_cross_call)(const struct cpumask *, unsigned int); - -void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) -{ - __smp_cross_call = fn; -} - static const char *ipi_types[NR_IPI] __tracepoint_string = { #define S(x,s) [x] = s S(IPI_RESCHEDULE, "Rescheduling interrupts"), @@ -800,11 +793,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { S(IPI_WAKEUP, "CPU wake-up interrupts"), }; -static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) -{ - trace_ipi_raise(target, ipi_types[ipinr]); - __smp_cross_call(target, ipinr); -} +static void smp_cross_call(const struct cpumask *target, unsigned int ipinr); void show_ipi_list(struct seq_file *p, int prec) { @@ -851,8 +840,7 @@ void arch_send_wakeup_ipi_mask(const struct cpumask *mask) #ifdef CONFIG_IRQ_WORK void arch_irq_work_raise(void) { - if (__smp_cross_call) - smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); + smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); } #endif @@ -959,34 +947,23 @@ static void do_handle_IPI(int ipinr) trace_ipi_exit_rcuidle(ipi_types[ipinr]); } -/* Legacy version, should go away once all irqchips have been converted */ -void handle_IPI(int ipinr, struct pt_regs *regs) -{ - struct pt_regs *old_regs = set_irq_regs(regs); - - irq_enter(); - do_handle_IPI(ipinr); - irq_exit(); - - set_irq_regs(old_regs); -} - static irqreturn_t ipi_handler(int irq, void *data) { do_handle_IPI(irq - ipi_irq_base); return IRQ_HANDLED; } -static void ipi_send(const struct cpumask *target, unsigned int ipi) +static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) { - __ipi_send_mask(ipi_desc[ipi], target); + trace_ipi_raise(target, ipi_types[ipinr]); + __ipi_send_mask(ipi_desc[ipinr], target); } static void ipi_setup(int cpu) { int i; - if (!ipi_irq_base) + if (WARN_ON_ONCE(!ipi_irq_base)) return; for (i = 0; i < nr_ipi; i++) @@ -997,7 +974,7 @@ static void ipi_teardown(int cpu) { int i; - if (!ipi_irq_base) + if (WARN_ON_ONCE(!ipi_irq_base)) return; for (i = 0; i < nr_ipi; i++) @@ -1023,7 +1000,6 @@ void __init set_smp_ipi_range(int ipi_base, int n) } ipi_irq_base = ipi_base; - __smp_cross_call = ipi_send; /* Setup the boot CPU immediately */ ipi_setup(smp_processor_id()); From a263881525310e10ecd46ae8e8531ac9e968b1b4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 20 Jun 2020 17:19:00 +0100 Subject: [PATCH 14/18] arm64: Remove custom IRQ stat accounting Let's switch the arm64 code to the core accounting, which already does everything we need. Reviewed-by: Valentin Schneider Acked-by: Catalin Marinas Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/hardirq.h | 9 --------- arch/arm64/include/asm/smp.h | 5 ----- arch/arm64/kernel/irq.c | 11 +---------- arch/arm64/kernel/smp.c | 30 ++++++++++++------------------ 4 files changed, 13 insertions(+), 42 deletions(-) diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h index 985493af704b..5ffa4bacdad3 100644 --- a/arch/arm64/include/asm/hardirq.h +++ b/arch/arm64/include/asm/hardirq.h @@ -13,21 +13,12 @@ #include #include -#define NR_IPI 7 - typedef struct { unsigned int __softirq_pending; - unsigned int ipi_irqs[NR_IPI]; } ____cacheline_aligned irq_cpustat_t; #include /* Standard mappings for irq_cpustat_t above */ -#define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++ -#define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member) - -u64 smp_irq_stat_cpu(unsigned int cpu); -#define arch_irq_stat_cpu smp_irq_stat_cpu - #define __ARCH_IRQ_EXIT_IRQS_DISABLED 1 struct nmi_ctx { diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index c298ad02252e..2e7f529ec5a6 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -55,11 +55,6 @@ static inline void set_cpu_logical_map(int cpu, u64 hwid) struct seq_file; -/* - * generate IPI list text - */ -extern void show_ipi_list(struct seq_file *p, int prec); - /* * Discover the set of possible CPUs and determine their * SMP operations. diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 04a327ccf84d..9cf2fb87584a 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -10,10 +10,10 @@ * Copyright (C) 2012 ARM Ltd. */ -#include #include #include #include +#include #include #include #include @@ -22,20 +22,11 @@ #include #include -unsigned long irq_err_count; - /* Only access this in an NMI enter/exit */ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); -int arch_show_interrupts(struct seq_file *p, int prec) -{ - show_ipi_list(p, prec); - seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count); - return 0; -} - #ifdef CONFIG_VMAP_STACK static void init_irq_stacks(void) { diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 58fb155fb0ab..b6bde2675ccc 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -72,7 +73,8 @@ enum ipi_msg_type { IPI_CPU_CRASH_STOP, IPI_TIMER, IPI_IRQ_WORK, - IPI_WAKEUP + IPI_WAKEUP, + NR_IPI }; static int ipi_irq_base __read_mostly; @@ -795,29 +797,23 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { static void smp_cross_call(const struct cpumask *target, unsigned int ipinr); -void show_ipi_list(struct seq_file *p, int prec) +unsigned long irq_err_count; + +int arch_show_interrupts(struct seq_file *p, int prec) { unsigned int cpu, i; for (i = 0; i < NR_IPI; i++) { + unsigned int irq = irq_desc_get_irq(ipi_desc[i]); seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); for_each_online_cpu(cpu) - seq_printf(p, "%10u ", - __get_irq_stat(cpu, ipi_irqs[i])); + seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu)); seq_printf(p, " %s\n", ipi_types[i]); } -} -u64 smp_irq_stat_cpu(unsigned int cpu) -{ - u64 sum = 0; - int i; - - for (i = 0; i < NR_IPI; i++) - sum += __get_irq_stat(cpu, ipi_irqs[i]); - - return sum; + seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count); + return 0; } void arch_send_call_function_ipi_mask(const struct cpumask *mask) @@ -892,10 +888,8 @@ static void do_handle_IPI(int ipinr) { unsigned int cpu = smp_processor_id(); - if ((unsigned)ipinr < NR_IPI) { + if ((unsigned)ipinr < NR_IPI) trace_ipi_entry_rcuidle(ipi_types[ipinr]); - __inc_irq_stat(cpu, ipi_irqs[ipinr]); - } switch (ipinr) { case IPI_RESCHEDULE: @@ -992,7 +986,7 @@ void __init set_smp_ipi_range(int ipi_base, int n) int err; err = request_percpu_irq(ipi_base + i, ipi_handler, - "IPI", &irq_stat); + "IPI", &cpu_number); WARN_ON(err); ipi_desc[i] = irq_to_desc(ipi_base + i); From 8aa837cb7a032884c787b15de81f7d9de8af0869 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 22 Jun 2020 22:15:54 +0100 Subject: [PATCH 15/18] ARM: Kill __smp_cross_call and co The old IPI registration interface is now unused on arm, so let's get rid of it. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- arch/arm/include/asm/smp.h | 6 ------ arch/arm/kernel/smp.c | 26 +++++++------------------- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index 0e29730295ca..0ca55a607d0a 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -39,12 +39,6 @@ void handle_IPI(int ipinr, struct pt_regs *regs); */ extern void smp_init_cpus(void); - -/* - * Provide a function to raise an IPI cross call on CPUs in callmap. - */ -extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); - /* * Register IPI interrupts with the arch SMP code */ diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index f21f78483353..d51e64955a26 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -511,14 +511,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) } } -static void (*__smp_cross_call)(const struct cpumask *, unsigned int); - -void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) -{ - if (!__smp_cross_call) - __smp_cross_call = fn; -} - static const char *ipi_types[NR_IPI] __tracepoint_string = { #define S(x,s) [x] = s S(IPI_WAKEUP, "CPU wakeup interrupts"), @@ -530,11 +522,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { S(IPI_COMPLETION, "completion interrupts"), }; -static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) -{ - trace_ipi_raise_rcuidle(target, ipi_types[ipinr]); - __smp_cross_call(target, ipinr); -} +static void smp_cross_call(const struct cpumask *target, unsigned int ipinr); void show_ipi_list(struct seq_file *p, int prec) { @@ -713,16 +701,17 @@ static irqreturn_t ipi_handler(int irq, void *data) return IRQ_HANDLED; } -static void ipi_send(const struct cpumask *target, unsigned int ipi) +static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) { - __ipi_send_mask(ipi_desc[ipi], target); + trace_ipi_raise_rcuidle(target, ipi_types[ipinr]); + __ipi_send_mask(ipi_desc[ipinr], target); } static void ipi_setup(int cpu) { int i; - if (!ipi_irq_base) + if (WARN_ON_ONCE(!ipi_irq_base)) return; for (i = 0; i < nr_ipi; i++) @@ -733,7 +722,7 @@ static void ipi_teardown(int cpu) { int i; - if (!ipi_irq_base) + if (WARN_ON_ONCE(!ipi_irq_base)) return; for (i = 0; i < nr_ipi; i++) @@ -759,7 +748,6 @@ void __init set_smp_ipi_range(int ipi_base, int n) } ipi_irq_base = ipi_base; - set_smp_cross_call(ipi_send); /* Setup the boot CPU immediately */ ipi_setup(smp_processor_id()); @@ -872,7 +860,7 @@ core_initcall(register_cpufreq_notifier); static void raise_nmi(cpumask_t *mask) { - __smp_cross_call(mask, IPI_CPU_BACKTRACE); + __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask); } void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) From 5ebf353af22c89d18964bb3b877a95200dfe07b9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 23 Jun 2020 21:15:00 +0100 Subject: [PATCH 16/18] ARM: Remove custom IRQ stat accounting Let's switch the arm code to the core accounting, which already does everything we need. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- arch/arm/include/asm/hardirq.h | 17 ----------------- arch/arm/kernel/irq.c | 1 - arch/arm/kernel/smp.c | 21 +++++---------------- 3 files changed, 5 insertions(+), 34 deletions(-) diff --git a/arch/arm/include/asm/hardirq.h b/arch/arm/include/asm/hardirq.h index 7a88f160b1fb..b95848ed2bc7 100644 --- a/arch/arm/include/asm/hardirq.h +++ b/arch/arm/include/asm/hardirq.h @@ -6,29 +6,12 @@ #include #include -/* number of IPIS _not_ including IPI_CPU_BACKTRACE */ -#define NR_IPI 7 - typedef struct { unsigned int __softirq_pending; -#ifdef CONFIG_SMP - unsigned int ipi_irqs[NR_IPI]; -#endif } ____cacheline_aligned irq_cpustat_t; #include /* Standard mappings for irq_cpustat_t above */ -#define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++ -#define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member) - -#ifdef CONFIG_SMP -u64 smp_irq_stat_cpu(unsigned int cpu); -#else -#define smp_irq_stat_cpu(cpu) 0 -#endif - -#define arch_irq_stat_cpu smp_irq_stat_cpu - #define __ARCH_IRQ_EXIT_IRQS_DISABLED 1 #endif /* __ASM_HARDIRQ_H */ diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index ee514034c0a1..698b6f636156 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -18,7 +18,6 @@ * IRQ's are in fact implemented a bit like signal handlers for the kernel. * Naturally it's not a 1:1 relation, but there are similarities. */ -#include #include #include #include diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index d51e64955a26..00327fa74b01 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -65,6 +66,7 @@ enum ipi_msg_type { IPI_CPU_STOP, IPI_IRQ_WORK, IPI_COMPLETION, + NR_IPI, /* * CPU_BACKTRACE is special and not included in NR_IPI * or tracable with trace_ipi_* @@ -529,27 +531,16 @@ void show_ipi_list(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < NR_IPI; i++) { + unsigned int irq = irq_desc_get_irq(ipi_desc[i]); seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); for_each_online_cpu(cpu) - seq_printf(p, "%10u ", - __get_irq_stat(cpu, ipi_irqs[i])); + seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu)); seq_printf(p, " %s\n", ipi_types[i]); } } -u64 smp_irq_stat_cpu(unsigned int cpu) -{ - u64 sum = 0; - int i; - - for (i = 0; i < NR_IPI; i++) - sum += __get_irq_stat(cpu, ipi_irqs[i]); - - return sum; -} - void arch_send_call_function_ipi_mask(const struct cpumask *mask) { smp_cross_call(mask, IPI_CALL_FUNC); @@ -630,10 +621,8 @@ static void do_handle_IPI(int ipinr) { unsigned int cpu = smp_processor_id(); - if ((unsigned)ipinr < NR_IPI) { + if ((unsigned)ipinr < NR_IPI) trace_ipi_entry_rcuidle(ipi_types[ipinr]); - __inc_irq_stat(cpu, ipi_irqs[ipinr]); - } switch (ipinr) { case IPI_WAKEUP: From c3330399931be38ce459e82bf7dea140338ae43f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 14 Sep 2020 17:21:16 +0100 Subject: [PATCH 17/18] irqchip/bcm2836: Provide mask/unmask dummy methods for IPIs Although it doesn't seem possible to disable individual mailbox interrupts, we still need to provide some callbacks. Fixes: 09eb672ce4fb ("irqchip/bcm2836: Configure mailbox interrupts as standard interrupts") Reported-by: Marek Szyprowski Tested-by: Marek Szyprowski Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-bcm2836.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/irqchip/irq-bcm2836.c b/drivers/irqchip/irq-bcm2836.c index 85df6ddad9be..97838eb705f9 100644 --- a/drivers/irqchip/irq-bcm2836.c +++ b/drivers/irqchip/irq-bcm2836.c @@ -193,6 +193,8 @@ static void bcm2836_arm_irqchip_ipi_send_mask(struct irq_data *d, static struct irq_chip bcm2836_arm_irqchip_ipi = { .name = "IPI", + .irq_mask = bcm2836_arm_irqchip_dummy_op, + .irq_unmask = bcm2836_arm_irqchip_dummy_op, .irq_eoi = bcm2836_arm_irqchip_ipi_eoi, .ipi_send_mask = bcm2836_arm_irqchip_ipi_send_mask, }; From 8594c3b85171b6f68e34e07b533ec2f1bf7fb065 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 15 Sep 2020 14:03:51 +0100 Subject: [PATCH 18/18] irqchip/gic: Cleanup Franken-GIC handling Introduce a static key identifying Samsung's unique creation, allowing to replace the indirect call to compute the base addresses with a simple test on the static key. Faster, cheaper, negative diffstat. Tested-by: Marek Szyprowski Signed-off-by: Marc Zyngier --- drivers/irqchip/irq-gic.c | 43 ++++++++++++--------------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 66671e1a6106..30edcca627d6 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -83,9 +83,6 @@ struct gic_chip_data { #endif struct irq_domain *domain; unsigned int gic_irqs; -#ifdef CONFIG_GIC_NON_BANKED - void __iomem *(*get_base)(union gic_base *); -#endif }; #ifdef CONFIG_BL_SWITCHER @@ -127,35 +124,27 @@ static struct gic_kvm_info gic_v2_kvm_info; static DEFINE_PER_CPU(u32, sgi_intid); #ifdef CONFIG_GIC_NON_BANKED -static void __iomem *gic_get_percpu_base(union gic_base *base) +static DEFINE_STATIC_KEY_FALSE(frankengic_key); + +static void enable_frankengic(void) { - return raw_cpu_read(*base->percpu_base); + static_branch_enable(&frankengic_key); } -static void __iomem *gic_get_common_base(union gic_base *base) +static inline void __iomem *__get_base(union gic_base *base) { + if (static_branch_unlikely(&frankengic_key)) + return raw_cpu_read(*base->percpu_base); + return base->common_base; } -static inline void __iomem *gic_data_dist_base(struct gic_chip_data *data) -{ - return data->get_base(&data->dist_base); -} - -static inline void __iomem *gic_data_cpu_base(struct gic_chip_data *data) -{ - return data->get_base(&data->cpu_base); -} - -static inline void gic_set_base_accessor(struct gic_chip_data *data, - void __iomem *(*f)(union gic_base *)) -{ - data->get_base = f; -} +#define gic_data_dist_base(d) __get_base(&(d)->dist_base) +#define gic_data_cpu_base(d) __get_base(&(d)->cpu_base) #else #define gic_data_dist_base(d) ((d)->dist_base.common_base) #define gic_data_cpu_base(d) ((d)->cpu_base.common_base) -#define gic_set_base_accessor(d, f) +#define enable_frankengic() do { } while(0) #endif static inline void __iomem *gic_dist_base(struct irq_data *d) @@ -307,7 +296,7 @@ static int gic_set_type(struct irq_data *d, unsigned int type) /* Interrupt configuration for SGIs can't be changed */ if (gicirq < 16) - return type == IRQ_TYPE_EDGE_RISING ? 0 : -EINVAL; + return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0; /* SPIs have restrictions on the supported types */ if (gicirq >= 32 && type != IRQ_TYPE_LEVEL_HIGH && @@ -720,11 +709,6 @@ static int gic_notifier(struct notifier_block *self, unsigned long cmd, void *v) int i; for (i = 0; i < CONFIG_ARM_GIC_MAX_NR; i++) { -#ifdef CONFIG_GIC_NON_BANKED - /* Skip over unused GICs */ - if (!gic_data[i].get_base) - continue; -#endif switch (cmd) { case CPU_PM_ENTER: gic_cpu_save(&gic_data[i]); @@ -1165,7 +1149,7 @@ static int gic_init_bases(struct gic_chip_data *gic, gic->raw_cpu_base + offset; } - gic_set_base_accessor(gic, gic_get_percpu_base); + enable_frankengic(); } else { /* Normal, sane GIC... */ WARN(gic->percpu_offset, @@ -1173,7 +1157,6 @@ static int gic_init_bases(struct gic_chip_data *gic, gic->percpu_offset); gic->dist_base.common_base = gic->raw_dist_base; gic->cpu_base.common_base = gic->raw_cpu_base; - gic_set_base_accessor(gic, gic_get_common_base); } /*