개념 설명 전체 · v6.18.37 / arch/arm64/kernel/stacktrace.c

    1 // SPDX-License-Identifier: GPL-2.0-only
    2 /*
    3  * Stack tracing support
    4  *
    5  * Copyright (C) 2012 ARM Ltd.
    6  */
    7 #include <linux/kernel.h>
    8 #include <linux/efi.h>
    9 #include <linux/export.h>
   10 #include <linux/filter.h>
   11 #include <linux/ftrace.h>
   12 #include <linux/kprobes.h>
   13 #include <linux/sched.h>
   14 #include <linux/sched/debug.h>
   15 #include <linux/sched/task_stack.h>
   16 #include <linux/stacktrace.h>
   17 
   18 #include <asm/efi.h>
   19 #include <asm/irq.h>
   20 #include <asm/stack_pointer.h>
   21 #include <asm/stacktrace.h>
   22 
   23 enum kunwind_source {
   24 	KUNWIND_SOURCE_UNKNOWN,
   25 	KUNWIND_SOURCE_FRAME,
   26 	KUNWIND_SOURCE_CALLER,
   27 	KUNWIND_SOURCE_TASK,
   28 	KUNWIND_SOURCE_REGS_PC,
   29 };
   30 
   31 union unwind_flags {
   32 	unsigned long	all;
   33 	struct {
   34 		unsigned long	fgraph : 1,
   35 				kretprobe : 1;
   36 	};
   37 };
   38 
   39 /*
   40  * Kernel unwind state
   41  *
   42  * @common:      Common unwind state.
   43  * @task:        The task being unwound.
   44  * @graph_idx:   Used by ftrace_graph_ret_addr() for optimized stack unwinding.
   45  * @kr_cur:      When KRETPROBES is selected, holds the kretprobe instance
   46  *               associated with the most recently encountered replacement lr
   47  *               value.
   48  */
   49 struct kunwind_state {
   50 	struct unwind_state common;
   51 	struct task_struct *task;
   52 	int graph_idx;
   53 #ifdef CONFIG_KRETPROBES
   54 	struct llist_node *kr_cur;
   55 #endif
   56 	enum kunwind_source source;
   57 	union unwind_flags flags;
   58 	struct pt_regs *regs;
   59 };
   60 
   61 static __always_inline void
   62 kunwind_init(struct kunwind_state *state,
   63 	     struct task_struct *task)
   64 {
   65 	unwind_init_common(&state->common);
   66 	state->task = task;
   67 	state->source = KUNWIND_SOURCE_UNKNOWN;
   68 	state->flags.all = 0;
   69 	state->regs = NULL;
   70 }
   71 
   72 /*
   73  * Start an unwind from a pt_regs.
   74  *
   75  * The unwind will begin at the PC within the regs.
   76  *
   77  * The regs must be on a stack currently owned by the calling task.
   78  */
   79 static __always_inline void
   80 kunwind_init_from_regs(struct kunwind_state *state,
   81 		       struct pt_regs *regs)
   82 {
   83 	kunwind_init(state, current);
   84 
   85 	state->regs = regs;
   86 	state->common.fp = regs->regs[29];
   87 	state->common.pc = regs->pc;
   88 	state->source = KUNWIND_SOURCE_REGS_PC;
   89 }
   90 
   91 /*
   92  * Start an unwind from a caller.
   93  *
   94  * The unwind will begin at the caller of whichever function this is inlined
   95  * into.
   96  *
   97  * The function which invokes this must be noinline.
   98  */
   99 static __always_inline void
  100 kunwind_init_from_caller(struct kunwind_state *state)
  101 {
  102 	kunwind_init(state, current);
  103 
  104 	state->common.fp = (unsigned long)__builtin_frame_address(1);
  105 	state->common.pc = (unsigned long)__builtin_return_address(0);
  106 	state->source = KUNWIND_SOURCE_CALLER;
  107 }
  108 
  109 /*
  110  * Start an unwind from a blocked task.
  111  *
  112  * The unwind will begin at the blocked tasks saved PC (i.e. the caller of
  113  * cpu_switch_to()).
  114  *
  115  * The caller should ensure the task is blocked in cpu_switch_to() for the
  116  * duration of the unwind, or the unwind will be bogus. It is never valid to
  117  * call this for the current task.
  118  */
  119 static __always_inline void
  120 kunwind_init_from_task(struct kunwind_state *state,
  121 		       struct task_struct *task)
  122 {
  123 	kunwind_init(state, task);
  124 
  125 	state->common.fp = thread_saved_fp(task);
  126 	state->common.pc = thread_saved_pc(task);
  127 	state->source = KUNWIND_SOURCE_TASK;
  128 }
  129 
  130 static __always_inline int
  131 kunwind_recover_return_address(struct kunwind_state *state)
  132 {
  133 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  134 	if (state->task->ret_stack &&
  135 	    (state->common.pc == (unsigned long)return_to_handler)) {
  136 		unsigned long orig_pc;
  137 		orig_pc = ftrace_graph_ret_addr(state->task, &state->graph_idx,
  138 						state->common.pc,
  139 						(void *)state->common.fp);
  140 		if (state->common.pc == orig_pc) {
  141 			WARN_ON_ONCE(state->task == current);
  142 			return -EINVAL;
  143 		}
  144 		state->common.pc = orig_pc;
  145 		state->flags.fgraph = 1;
  146 	}
  147 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
  148 
  149 #ifdef CONFIG_KRETPROBES
  150 	if (is_kretprobe_trampoline(state->common.pc)) {
  151 		unsigned long orig_pc;
  152 		orig_pc = kretprobe_find_ret_addr(state->task,
  153 						  (void *)state->common.fp,
  154 						  &state->kr_cur);
  155 		if (!orig_pc)
  156 			return -EINVAL;
  157 		state->common.pc = orig_pc;
  158 		state->flags.kretprobe = 1;
  159 	}
  160 #endif /* CONFIG_KRETPROBES */
  161 
  162 	return 0;
  163 }
  164 
  165 static __always_inline
  166 int kunwind_next_regs_pc(struct kunwind_state *state)
  167 {
  168 	struct stack_info *info;
  169 	unsigned long fp = state->common.fp;
  170 	struct pt_regs *regs;
  171 
  172 	regs = container_of((u64 *)fp, struct pt_regs, stackframe.record.fp);
  173 
  174 	info = unwind_find_stack(&state->common, (unsigned long)regs, sizeof(*regs));
  175 	if (!info)
  176 		return -EINVAL;
  177 
  178 	unwind_consume_stack(&state->common, info, (unsigned long)regs,
  179 			     sizeof(*regs));
  180 
  181 	state->regs = regs;
  182 	state->common.pc = regs->pc;
  183 	state->common.fp = regs->regs[29];
  184 	state->regs = NULL;
  185 	state->source = KUNWIND_SOURCE_REGS_PC;
  186 	return 0;
  187 }
  188 
  189 static __always_inline int
  190 kunwind_next_frame_record_meta(struct kunwind_state *state)
  191 {
  192 	struct task_struct *tsk = state->task;
  193 	unsigned long fp = state->common.fp;
  194 	struct frame_record_meta *meta;
  195 	struct stack_info *info;
  196 
  197 	info = unwind_find_stack(&state->common, fp, sizeof(*meta));
  198 	if (!info)
  199 		return -EINVAL;
  200 
  201 	meta = (struct frame_record_meta *)fp;
  202 	switch (READ_ONCE(meta->type)) {
  203 	case FRAME_META_TYPE_FINAL:
  204 		if (meta == &task_pt_regs(tsk)->stackframe)
  205 			return -ENOENT;
  206 		WARN_ON_ONCE(tsk == current);
  207 		return -EINVAL;
  208 	case FRAME_META_TYPE_PT_REGS:
  209 		return kunwind_next_regs_pc(state);
  210 	default:
  211 		WARN_ON_ONCE(tsk == current);
  212 		return -EINVAL;
  213 	}
  214 }
  215 
  216 static __always_inline int
  217 kunwind_next_frame_record(struct kunwind_state *state)
  218 {
  219 	unsigned long fp = state->common.fp;
  220 	struct frame_record *record;
  221 	struct stack_info *info;
  222 	unsigned long new_fp, new_pc;
  223 
  224 	if (fp & 0x7)
  225 		return -EINVAL;
  226 
  227 	info = unwind_find_stack(&state->common, fp, sizeof(*record));
  228 	if (!info)
  229 		return -EINVAL;
  230 
  231 	record = (struct frame_record *)fp;
  232 	new_fp = READ_ONCE(record->fp);
  233 	new_pc = READ_ONCE(record->lr);
  234 
  235 	if (!new_fp && !new_pc)
  236 		return kunwind_next_frame_record_meta(state);
  237 
  238 	unwind_consume_stack(&state->common, info, fp, sizeof(*record));
  239 
  240 	state->common.fp = new_fp;
  241 	state->common.pc = new_pc;
  242 	state->source = KUNWIND_SOURCE_FRAME;
  243 
  244 	return 0;
  245 }
  246 
  247 /*
  248  * Unwind from one frame record (A) to the next frame record (B).
  249  *
  250  * We terminate early if the location of B indicates a malformed chain of frame
  251  * records (e.g. a cycle), determined based on the location and fp value of A
  252  * and the location (but not the fp value) of B.
  253  */
  254 static __always_inline int
  255 kunwind_next(struct kunwind_state *state)
  256 {
  257 	int err;
  258 
  259 	state->flags.all = 0;
  260 
  261 	switch (state->source) {
  262 	case KUNWIND_SOURCE_FRAME:
  263 	case KUNWIND_SOURCE_CALLER:
  264 	case KUNWIND_SOURCE_TASK:
  265 	case KUNWIND_SOURCE_REGS_PC:
  266 		err = kunwind_next_frame_record(state);
  267 		break;
  268 	default:
  269 		err = -EINVAL;
  270 	}
  271 
  272 	if (err)
  273 		return err;
  274 
  275 	state->common.pc = ptrauth_strip_kernel_insn_pac(state->common.pc);
  276 
  277 	return kunwind_recover_return_address(state);
  278 }
  279 
  280 typedef bool (*kunwind_consume_fn)(const struct kunwind_state *state, void *cookie);
  281 
  282 static __always_inline int
  283 do_kunwind(struct kunwind_state *state, kunwind_consume_fn consume_state,
  284 	   void *cookie)
  285 {
  286 	int ret;
  287 
  288 	ret = kunwind_recover_return_address(state);
  289 	if (ret)
  290 		return ret;
  291 
  292 	while (1) {
  293 		if (!consume_state(state, cookie))
  294 			return -EINVAL;
  295 		ret = kunwind_next(state);
  296 		if (ret == -ENOENT)
  297 			return 0;
  298 		if (ret < 0)
  299 			return ret;
  300 	}
  301 }
  302 
  303 /*
  304  * Per-cpu stacks are only accessible when unwinding the current task in a
  305  * non-preemptible context.
  306  */
  307 #define STACKINFO_CPU(name)					\
  308 	({							\
  309 		((task == current) && !preemptible())		\
  310 			? stackinfo_get_##name()		\
  311 			: stackinfo_get_unknown();		\
  312 	})
  313 
  314 /*
  315  * SDEI stacks are only accessible when unwinding the current task in an NMI
  316  * context.
  317  */
  318 #define STACKINFO_SDEI(name)					\
  319 	({							\
  320 		((task == current) && in_nmi())			\
  321 			? stackinfo_get_sdei_##name()		\
  322 			: stackinfo_get_unknown();		\
  323 	})
  324 
  325 #define STACKINFO_EFI						\
  326 	({							\
  327 		((task == current) && current_in_efi())		\
  328 			? stackinfo_get_efi()			\
  329 			: stackinfo_get_unknown();		\
  330 	})
  331 
  332 static __always_inline int
  333 kunwind_stack_walk(kunwind_consume_fn consume_state,
  334 		   void *cookie, struct task_struct *task,
  335 		   struct pt_regs *regs)
  336 {
  337 	struct stack_info stacks[] = {
  338 		stackinfo_get_task(task),
  339 		STACKINFO_CPU(irq),
  340 		STACKINFO_CPU(overflow),
  341 #if defined(CONFIG_ARM_SDE_INTERFACE)
  342 		STACKINFO_SDEI(normal),
  343 		STACKINFO_SDEI(critical),
  344 #endif
  345 #ifdef CONFIG_EFI
  346 		STACKINFO_EFI,
  347 #endif
  348 	};
  349 	struct kunwind_state state = {
  350 		.common = {
  351 			.stacks = stacks,
  352 			.nr_stacks = ARRAY_SIZE(stacks),
  353 		},
  354 	};
  355 
  356 	if (regs) {
  357 		if (task != current)
  358 			return -EINVAL;
  359 		kunwind_init_from_regs(&state, regs);
  360 	} else if (task == current) {
  361 		kunwind_init_from_caller(&state);
  362 	} else {
  363 		kunwind_init_from_task(&state, task);
  364 	}
  365 
  366 	return do_kunwind(&state, consume_state, cookie);
  367 }
  368 
  369 struct kunwind_consume_entry_data {
  370 	stack_trace_consume_fn consume_entry;
  371 	void *cookie;
  372 };
  373 
  374 static __always_inline bool
  375 arch_kunwind_consume_entry(const struct kunwind_state *state, void *cookie)
  376 {
  377 	struct kunwind_consume_entry_data *data = cookie;
  378 	return data->consume_entry(data->cookie, state->common.pc);
  379 }
  380 
  381 noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry,
  382 			      void *cookie, struct task_struct *task,
  383 			      struct pt_regs *regs)
  384 {
  385 	struct kunwind_consume_entry_data data = {
  386 		.consume_entry = consume_entry,
  387 		.cookie = cookie,
  388 	};
  389 
  390 	kunwind_stack_walk(arch_kunwind_consume_entry, &data, task, regs);
  391 }
  392 
  393 static __always_inline bool
  394 arch_reliable_kunwind_consume_entry(const struct kunwind_state *state, void *cookie)
  395 {
  396 	/*
  397 	 * At an exception boundary we can reliably consume the saved PC. We do
  398 	 * not know whether the LR was live when the exception was taken, and
  399 	 * so we cannot perform the next unwind step reliably.
  400 	 *
  401 	 * All that matters is whether the *entire* unwind is reliable, so give
  402 	 * up as soon as we hit an exception boundary.
  403 	 */
  404 	if (state->source == KUNWIND_SOURCE_REGS_PC)
  405 		return false;
  406 
  407 	return arch_kunwind_consume_entry(state, cookie);
  408 }
  409 
  410 noinline noinstr int arch_stack_walk_reliable(stack_trace_consume_fn consume_entry,
  411 					      void *cookie,
  412 					      struct task_struct *task)
  413 {
  414 	struct kunwind_consume_entry_data data = {
  415 		.consume_entry = consume_entry,
  416 		.cookie = cookie,
  417 	};
  418 
  419 	return kunwind_stack_walk(arch_reliable_kunwind_consume_entry, &data,
  420 				  task, NULL);
  421 }
  422 
  423 struct bpf_unwind_consume_entry_data {
  424 	bool (*consume_entry)(void *cookie, u64 ip, u64 sp, u64 fp);
  425 	void *cookie;
  426 };
  427 
  428 static bool
  429 arch_bpf_unwind_consume_entry(const struct kunwind_state *state, void *cookie)
  430 {
  431 	struct bpf_unwind_consume_entry_data *data = cookie;
  432 
  433 	return data->consume_entry(data->cookie, state->common.pc, 0,
  434 				   state->common.fp);
  435 }
  436 
  437 noinline noinstr void arch_bpf_stack_walk(bool (*consume_entry)(void *cookie, u64 ip, u64 sp,
  438 								u64 fp), void *cookie)
  439 {
  440 	struct bpf_unwind_consume_entry_data data = {
  441 		.consume_entry = consume_entry,
  442 		.cookie = cookie,
  443 	};
  444 
  445 	kunwind_stack_walk(arch_bpf_unwind_consume_entry, &data, current, NULL);
  446 }
  447 
  448 static const char *state_source_string(const struct kunwind_state *state)
  449 {
  450 	switch (state->source) {
  451 	case KUNWIND_SOURCE_FRAME:	return NULL;
  452 	case KUNWIND_SOURCE_CALLER:	return "C";
  453 	case KUNWIND_SOURCE_TASK:	return "T";
  454 	case KUNWIND_SOURCE_REGS_PC:	return "P";
  455 	default:			return "U";
  456 	}
  457 }
  458 
  459 static bool dump_backtrace_entry(const struct kunwind_state *state, void *arg)
  460 {
  461 	const char *source = state_source_string(state);
  462 	union unwind_flags flags = state->flags;
  463 	bool has_info = source || flags.all;
  464 	char *loglvl = arg;
  465 
  466 	printk("%s %pSb%s%s%s%s%s\n", loglvl,
  467 		(void *)state->common.pc,
  468 		has_info ? " (" : "",
  469 		source ? source : "",
  470 		flags.fgraph ? "F" : "",
  471 		flags.kretprobe ? "K" : "",
  472 		has_info ? ")" : "");
  473 
  474 	return true;
  475 }
  476 
  477 void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
  478 		    const char *loglvl)
  479 {
  480 	pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
  481 
  482 	if (regs && user_mode(regs))
  483 		return;
  484 
  485 	if (!tsk)
  486 		tsk = current;
  487 
  488 	if (!try_get_task_stack(tsk))
  489 		return;
  490 
  491 	printk("%sCall trace:\n", loglvl);
  492 	kunwind_stack_walk(dump_backtrace_entry, (void *)loglvl, tsk, regs);
  493 
  494 	put_task_stack(tsk);
  495 }
  496 
  497 void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
  498 {
  499 	dump_backtrace(NULL, tsk, loglvl);
  500 	barrier();
  501 }
  502 
  503 /*
  504  * The struct defined for userspace stack frame in AARCH64 mode.
  505  */
  506 struct frame_tail {
  507 	struct frame_tail	__user *fp;
  508 	unsigned long		lr;
  509 } __attribute__((packed));
  510 
  511 /*
  512  * Get the return address for a single stackframe and return a pointer to the
  513  * next frame tail.
  514  */
  515 static struct frame_tail __user *
  516 unwind_user_frame(struct frame_tail __user *tail, void *cookie,
  517 	       stack_trace_consume_fn consume_entry)
  518 {
  519 	struct frame_tail buftail;
  520 	unsigned long err;
  521 	unsigned long lr;
  522 
  523 	/* Also check accessibility of one struct frame_tail beyond */
  524 	if (!access_ok(tail, sizeof(buftail)))
  525 		return NULL;
  526 
  527 	pagefault_disable();
  528 	err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
  529 	pagefault_enable();
  530 
  531 	if (err)
  532 		return NULL;
  533 
  534 	lr = ptrauth_strip_user_insn_pac(buftail.lr);
  535 
  536 	if (!consume_entry(cookie, lr))
  537 		return NULL;
  538 
  539 	/*
  540 	 * Frame pointers should strictly progress back up the stack
  541 	 * (towards higher addresses).
  542 	 */
  543 	if (tail >= buftail.fp)
  544 		return NULL;
  545 
  546 	return buftail.fp;
  547 }
  548 
  549 #ifdef CONFIG_COMPAT
  550 /*
  551  * The registers we're interested in are at the end of the variable
  552  * length saved register structure. The fp points at the end of this
  553  * structure so the address of this struct is:
  554  * (struct compat_frame_tail *)(xxx->fp)-1
  555  *
  556  * This code has been adapted from the ARM OProfile support.
  557  */
  558 struct compat_frame_tail {
  559 	compat_uptr_t	fp; /* a (struct compat_frame_tail *) in compat mode */
  560 	u32		sp;
  561 	u32		lr;
  562 } __attribute__((packed));
  563 
  564 static struct compat_frame_tail __user *
  565 unwind_compat_user_frame(struct compat_frame_tail __user *tail, void *cookie,
  566 				stack_trace_consume_fn consume_entry)
  567 {
  568 	struct compat_frame_tail buftail;
  569 	unsigned long err;
  570 
  571 	/* Also check accessibility of one struct frame_tail beyond */
  572 	if (!access_ok(tail, sizeof(buftail)))
  573 		return NULL;
  574 
  575 	pagefault_disable();
  576 	err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
  577 	pagefault_enable();
  578 
  579 	if (err)
  580 		return NULL;
  581 
  582 	if (!consume_entry(cookie, buftail.lr))
  583 		return NULL;
  584 
  585 	/*
  586 	 * Frame pointers should strictly progress back up the stack
  587 	 * (towards higher addresses).
  588 	 */
  589 	if (tail + 1 >= (struct compat_frame_tail __user *)
  590 			compat_ptr(buftail.fp))
  591 		return NULL;
  592 
  593 	return (struct compat_frame_tail __user *)compat_ptr(buftail.fp) - 1;
  594 }
  595 #endif /* CONFIG_COMPAT */
  596 
  597 
  598 void arch_stack_walk_user(stack_trace_consume_fn consume_entry, void *cookie,
  599 					const struct pt_regs *regs)
  600 {
  601 	if (!consume_entry(cookie, regs->pc))
  602 		return;
  603 
  604 	if (!compat_user_mode(regs)) {
  605 		/* AARCH64 mode */
  606 		struct frame_tail __user *tail;
  607 
  608 		tail = (struct frame_tail __user *)regs->regs[29];
  609 		while (tail && !((unsigned long)tail & 0x7))
  610 			tail = unwind_user_frame(tail, cookie, consume_entry);
  611 	} else {
  612 #ifdef CONFIG_COMPAT
  613 		/* AARCH32 compat mode */
  614 		struct compat_frame_tail __user *tail;
  615 
  616 		tail = (struct compat_frame_tail __user *)regs->compat_fp - 1;
  617 		while (tail && !((unsigned long)tail & 0x3))
  618 			tail = unwind_compat_user_frame(tail, cookie, consume_entry);
  619 #endif
  620 	}
  621 }