개념 설명 전체 · v6.18.37 / arch/riscv/kernel/smp.c

    1 // SPDX-License-Identifier: GPL-2.0-only
    2 /*
    3  * SMP initialisation and IPI support
    4  * Based on arch/arm64/kernel/smp.c
    5  *
    6  * Copyright (C) 2012 ARM Ltd.
    7  * Copyright (C) 2015 Regents of the University of California
    8  * Copyright (C) 2017 SiFive
    9  */
   10 
   11 #include <linux/cpu.h>
   12 #include <linux/clockchips.h>
   13 #include <linux/interrupt.h>
   14 #include <linux/module.h>
   15 #include <linux/kexec.h>
   16 #include <linux/kgdb.h>
   17 #include <linux/percpu.h>
   18 #include <linux/profile.h>
   19 #include <linux/smp.h>
   20 #include <linux/sched.h>
   21 #include <linux/seq_file.h>
   22 #include <linux/delay.h>
   23 #include <linux/irq.h>
   24 #include <linux/irq_work.h>
   25 #include <linux/nmi.h>
   26 
   27 #include <asm/tlbflush.h>
   28 #include <asm/cacheflush.h>
   29 #include <asm/cpu_ops.h>
   30 
   31 enum ipi_message_type {
   32 	IPI_RESCHEDULE,
   33 	IPI_CALL_FUNC,
   34 	IPI_CPU_STOP,
   35 	IPI_CPU_CRASH_STOP,
   36 	IPI_IRQ_WORK,
   37 	IPI_TIMER,
   38 	IPI_CPU_BACKTRACE,
   39 	IPI_KGDB_ROUNDUP,
   40 	IPI_MAX
   41 };
   42 
   43 static const char * const ipi_names[] = {
   44 	[IPI_RESCHEDULE]	= "Rescheduling interrupts",
   45 	[IPI_CALL_FUNC]		= "Function call interrupts",
   46 	[IPI_CPU_STOP]		= "CPU stop interrupts",
   47 	[IPI_CPU_CRASH_STOP]	= "CPU stop (for crash dump) interrupts",
   48 	[IPI_IRQ_WORK]		= "IRQ work interrupts",
   49 	[IPI_TIMER]		= "Timer broadcast interrupts",
   50 	[IPI_CPU_BACKTRACE]     = "CPU backtrace interrupts",
   51 	[IPI_KGDB_ROUNDUP]	= "KGDB roundup interrupts",
   52 };
   53 
   54 unsigned long __cpuid_to_hartid_map[NR_CPUS] __ro_after_init = {
   55 	[0 ... NR_CPUS-1] = INVALID_HARTID
   56 };
   57 EXPORT_SYMBOL_GPL(__cpuid_to_hartid_map);
   58 
   59 void __init smp_setup_processor_id(void)
   60 {
   61 	cpuid_to_hartid_map(0) = boot_cpu_hartid;
   62 
   63 	pr_info("Booting Linux on hartid %lu\n", boot_cpu_hartid);
   64 }
   65 
   66 static DEFINE_PER_CPU_READ_MOSTLY(int, ipi_dummy_dev);
   67 static int ipi_virq_base __ro_after_init;
   68 static int nr_ipi __ro_after_init = IPI_MAX;
   69 static struct irq_desc *ipi_desc[IPI_MAX] __read_mostly;
   70 
   71 int riscv_hartid_to_cpuid(unsigned long hartid)
   72 {
   73 	int i;
   74 
   75 	for (i = 0; i < NR_CPUS; i++)
   76 		if (cpuid_to_hartid_map(i) == hartid)
   77 			return i;
   78 
   79 	return -ENOENT;
   80 }
   81 
   82 static void ipi_stop(void)
   83 {
   84 	set_cpu_online(smp_processor_id(), false);
   85 	while (1)
   86 		wait_for_interrupt();
   87 }
   88 
   89 #ifdef CONFIG_KEXEC_CORE
   90 static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0);
   91 
   92 static inline void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
   93 {
   94 	crash_save_cpu(regs, cpu);
   95 
   96 	atomic_dec(&waiting_for_crash_ipi);
   97 
   98 	local_irq_disable();
   99 
  100 #ifdef CONFIG_HOTPLUG_CPU
  101 	if (cpu_has_hotplug(cpu))
  102 		cpu_ops->cpu_stop();
  103 #endif
  104 
  105 	for(;;)
  106 		wait_for_interrupt();
  107 }
  108 #else
  109 static inline void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
  110 {
  111 	unreachable();
  112 }
  113 #endif
  114 
  115 static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op)
  116 {
  117 	__ipi_send_mask(ipi_desc[op], mask);
  118 }
  119 
  120 static void send_ipi_single(int cpu, enum ipi_message_type op)
  121 {
  122 	__ipi_send_mask(ipi_desc[op], cpumask_of(cpu));
  123 }
  124 
  125 #ifdef CONFIG_IRQ_WORK
  126 void arch_irq_work_raise(void)
  127 {
  128 	send_ipi_single(smp_processor_id(), IPI_IRQ_WORK);
  129 }
  130 #endif
  131 
  132 static irqreturn_t handle_IPI(int irq, void *data)
  133 {
  134 	unsigned int cpu = smp_processor_id();
  135 	int ipi = irq - ipi_virq_base;
  136 
  137 	switch (ipi) {
  138 	case IPI_RESCHEDULE:
  139 		scheduler_ipi();
  140 		break;
  141 	case IPI_CALL_FUNC:
  142 		generic_smp_call_function_interrupt();
  143 		break;
  144 	case IPI_CPU_STOP:
  145 		ipi_stop();
  146 		break;
  147 	case IPI_CPU_CRASH_STOP:
  148 		ipi_cpu_crash_stop(cpu, get_irq_regs());
  149 		break;
  150 	case IPI_IRQ_WORK:
  151 		irq_work_run();
  152 		break;
  153 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
  154 	case IPI_TIMER:
  155 		tick_receive_broadcast();
  156 		break;
  157 #endif
  158 	case IPI_CPU_BACKTRACE:
  159 		nmi_cpu_backtrace(get_irq_regs());
  160 		break;
  161 	case IPI_KGDB_ROUNDUP:
  162 		kgdb_nmicallback(cpu, get_irq_regs());
  163 		break;
  164 	default:
  165 		pr_warn("CPU%d: unhandled IPI%d\n", cpu, ipi);
  166 		break;
  167 	}
  168 
  169 	return IRQ_HANDLED;
  170 }
  171 
  172 void riscv_ipi_enable(void)
  173 {
  174 	int i;
  175 
  176 	if (WARN_ON_ONCE(!ipi_virq_base))
  177 		return;
  178 
  179 	for (i = 0; i < nr_ipi; i++)
  180 		enable_percpu_irq(ipi_virq_base + i, 0);
  181 }
  182 
  183 void riscv_ipi_disable(void)
  184 {
  185 	int i;
  186 
  187 	if (WARN_ON_ONCE(!ipi_virq_base))
  188 		return;
  189 
  190 	for (i = 0; i < nr_ipi; i++)
  191 		disable_percpu_irq(ipi_virq_base + i);
  192 }
  193 
  194 bool riscv_ipi_have_virq_range(void)
  195 {
  196 	return (ipi_virq_base) ? true : false;
  197 }
  198 
  199 void riscv_ipi_set_virq_range(int virq, int nr)
  200 {
  201 	int i, err;
  202 
  203 	if (WARN_ON(ipi_virq_base))
  204 		return;
  205 
  206 	WARN_ON(nr < IPI_MAX);
  207 	nr_ipi = min(nr, IPI_MAX);
  208 	ipi_virq_base = virq;
  209 
  210 	/* Request IPIs */
  211 	for (i = 0; i < nr_ipi; i++) {
  212 		err = request_percpu_irq(ipi_virq_base + i, handle_IPI,
  213 					 ipi_names[i], &ipi_dummy_dev);
  214 		WARN_ON(err);
  215 
  216 		ipi_desc[i] = irq_to_desc(ipi_virq_base + i);
  217 		irq_set_status_flags(ipi_virq_base + i, IRQ_HIDDEN);
  218 	}
  219 
  220 	/* Enabled IPIs for boot CPU immediately */
  221 	riscv_ipi_enable();
  222 }
  223 
  224 void show_ipi_stats(struct seq_file *p, int prec)
  225 {
  226 	unsigned int cpu, i;
  227 
  228 	for (i = 0; i < IPI_MAX; i++) {
  229 		seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
  230 			   prec >= 4 ? " " : "");
  231 		for_each_online_cpu(cpu)
  232 			seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
  233 		seq_printf(p, " %s\n", ipi_names[i]);
  234 	}
  235 }
  236 
  237 void arch_send_call_function_ipi_mask(struct cpumask *mask)
  238 {
  239 	send_ipi_mask(mask, IPI_CALL_FUNC);
  240 }
  241 
  242 void arch_send_call_function_single_ipi(int cpu)
  243 {
  244 	send_ipi_single(cpu, IPI_CALL_FUNC);
  245 }
  246 
  247 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
  248 void tick_broadcast(const struct cpumask *mask)
  249 {
  250 	send_ipi_mask(mask, IPI_TIMER);
  251 }
  252 #endif
  253 
  254 void smp_send_stop(void)
  255 {
  256 	unsigned long timeout;
  257 
  258 	if (num_online_cpus() > 1) {
  259 		cpumask_t mask;
  260 
  261 		cpumask_copy(&mask, cpu_online_mask);
  262 		cpumask_clear_cpu(smp_processor_id(), &mask);
  263 
  264 		if (system_state <= SYSTEM_RUNNING)
  265 			pr_crit("SMP: stopping secondary CPUs\n");
  266 		send_ipi_mask(&mask, IPI_CPU_STOP);
  267 	}
  268 
  269 	/* Wait up to one second for other CPUs to stop */
  270 	timeout = USEC_PER_SEC;
  271 	while (num_online_cpus() > 1 && timeout--)
  272 		udelay(1);
  273 
  274 	if (num_online_cpus() > 1)
  275 		pr_warn("SMP: failed to stop secondary CPUs %*pbl\n",
  276 			   cpumask_pr_args(cpu_online_mask));
  277 }
  278 
  279 #ifdef CONFIG_KEXEC_CORE
  280 /*
  281  * The number of CPUs online, not counting this CPU (which may not be
  282  * fully online and so not counted in num_online_cpus()).
  283  */
  284 static inline unsigned int num_other_online_cpus(void)
  285 {
  286 	unsigned int this_cpu_online = cpu_online(smp_processor_id());
  287 
  288 	return num_online_cpus() - this_cpu_online;
  289 }
  290 
  291 void crash_smp_send_stop(void)
  292 {
  293 	static int cpus_stopped;
  294 	cpumask_t mask;
  295 	unsigned long timeout;
  296 
  297 	/*
  298 	 * This function can be called twice in panic path, but obviously
  299 	 * we execute this only once.
  300 	 */
  301 	if (cpus_stopped)
  302 		return;
  303 
  304 	cpus_stopped = 1;
  305 
  306 	/*
  307 	 * If this cpu is the only one alive at this point in time, online or
  308 	 * not, there are no stop messages to be sent around, so just back out.
  309 	 */
  310 	if (num_other_online_cpus() == 0)
  311 		return;
  312 
  313 	cpumask_copy(&mask, cpu_online_mask);
  314 	cpumask_clear_cpu(smp_processor_id(), &mask);
  315 
  316 	atomic_set(&waiting_for_crash_ipi, num_other_online_cpus());
  317 
  318 	pr_crit("SMP: stopping secondary CPUs\n");
  319 	send_ipi_mask(&mask, IPI_CPU_CRASH_STOP);
  320 
  321 	/* Wait up to one second for other CPUs to stop */
  322 	timeout = USEC_PER_SEC;
  323 	while ((atomic_read(&waiting_for_crash_ipi) > 0) && timeout--)
  324 		udelay(1);
  325 
  326 	if (atomic_read(&waiting_for_crash_ipi) > 0)
  327 		pr_warn("SMP: failed to stop secondary CPUs %*pbl\n",
  328 			cpumask_pr_args(&mask));
  329 }
  330 
  331 bool smp_crash_stop_failed(void)
  332 {
  333 	return (atomic_read(&waiting_for_crash_ipi) > 0);
  334 }
  335 #endif
  336 
  337 void arch_smp_send_reschedule(int cpu)
  338 {
  339 	send_ipi_single(cpu, IPI_RESCHEDULE);
  340 }
  341 EXPORT_SYMBOL_GPL(arch_smp_send_reschedule);
  342 
  343 static void riscv_backtrace_ipi(cpumask_t *mask)
  344 {
  345 	send_ipi_mask(mask, IPI_CPU_BACKTRACE);
  346 }
  347 
  348 void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)
  349 {
  350 	nmi_trigger_cpumask_backtrace(mask, exclude_cpu, riscv_backtrace_ipi);
  351 }
  352 
  353 #ifdef CONFIG_KGDB
  354 void kgdb_roundup_cpus(void)
  355 {
  356 	int this_cpu = raw_smp_processor_id();
  357 	int cpu;
  358 
  359 	for_each_online_cpu(cpu) {
  360 		/* No need to roundup ourselves */
  361 		if (cpu == this_cpu)
  362 			continue;
  363 
  364 		send_ipi_single(cpu, IPI_KGDB_ROUNDUP);
  365 	}
  366 }
  367 #endif