mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-10 12:34:07 +08:00
a625357997
Sometimes it is necessary to use a PLT entry to call an ftrace trampoline. This is handled by ftrace_make_call() and ftrace_make_nop(), with each having *almost* identical logic, but this is not handled by ftrace_modify_call() since its introduction in commit:3b23e4991f
("arm64: implement ftrace with regs") Due to this, if we ever were to call ftrace_modify_call() for a callsite which requires a PLT entry for a trampoline, then either: a) If the old addr requires a trampoline, ftrace_modify_call() will use an out-of-range address to generate the 'old' branch instruction. This will result in warnings from aarch64_insn_gen_branch_imm() and ftrace_modify_code(), and no instructions will be modified. As ftrace_modify_call() will return an error, this will result in subsequent internal ftrace errors. b) If the old addr does not require a trampoline, but the new addr does, ftrace_modify_call() will use an out-of-range address to generate the 'new' branch instruction. This will result in warnings from aarch64_insn_gen_branch_imm(), and ftrace_modify_code() will replace the 'old' branch with a BRK. This will result in a kernel panic when this BRK is later executed. Practically speaking, case (a) is vastly more likely than case (b), and typically this will result in internal ftrace errors that don't necessarily affect the rest of the system. This can be demonstrated with an out-of-tree test module which triggers ftrace_modify_call(), e.g. | # insmod test_ftrace.ko | test_ftrace: Function test_function raw=0xffffb3749399201c, callsite=0xffffb37493992024 | branch_imm_common: offset out of range | branch_imm_common: offset out of range | ------------[ ftrace bug ]------------ | ftrace failed to modify | [<ffffb37493992024>] test_function+0x8/0x38 [test_ftrace] | actual: 1d:00:00:94 | Updating ftrace call site to call a different ftrace function | ftrace record flags: e0000002 | (2) R | expected tramp: ffffb374ae42ed54 | ------------[ cut here ]------------ | WARNING: CPU: 0 PID: 165 at kernel/trace/ftrace.c:2085 ftrace_bug+0x280/0x2b0 | Modules linked in: test_ftrace(+) | CPU: 0 PID: 165 Comm: insmod Not tainted 5.19.0-rc2-00002-g4d9ead8b45ce #13 | Hardware name: linux,dummy-virt (DT) | pstate: 60400005 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) | pc : ftrace_bug+0x280/0x2b0 | lr : ftrace_bug+0x280/0x2b0 | sp : ffff80000839ba00 | x29: ffff80000839ba00 x28: 0000000000000000 x27: ffff80000839bcf0 | x26: ffffb37493994180 x25: ffffb374b0991c28 x24: ffffb374b0d70000 | x23: 00000000ffffffea x22: ffffb374afcc33b0 x21: ffffb374b08f9cc8 | x20: ffff572b8462c000 x19: ffffb374b08f9000 x18: ffffffffffffffff | x17: 6c6c6163202c6331 x16: ffffb374ae5ad110 x15: ffffb374b0d51ee4 | x14: 0000000000000000 x13: 3435646532346561 x12: 3437336266666666 | x11: 203a706d61727420 x10: 6465746365707865 x9 : ffffb374ae5149e8 | x8 : 336266666666203a x7 : 706d617274206465 x6 : 00000000fffff167 | x5 : ffff572bffbc4a08 x4 : 00000000fffff167 x3 : 0000000000000000 | x2 : 0000000000000000 x1 : ffff572b84461e00 x0 : 0000000000000022 | Call trace: | ftrace_bug+0x280/0x2b0 | ftrace_replace_code+0x98/0xa0 | ftrace_modify_all_code+0xe0/0x144 | arch_ftrace_update_code+0x14/0x20 | ftrace_startup+0xf8/0x1b0 | register_ftrace_function+0x38/0x90 | test_ftrace_init+0xd0/0x1000 [test_ftrace] | do_one_initcall+0x50/0x2b0 | do_init_module+0x50/0x1f0 | load_module+0x17c8/0x1d64 | __do_sys_finit_module+0xa8/0x100 | __arm64_sys_finit_module+0x2c/0x3c | invoke_syscall+0x50/0x120 | el0_svc_common.constprop.0+0xdc/0x100 | do_el0_svc+0x3c/0xd0 | el0_svc+0x34/0xb0 | el0t_64_sync_handler+0xbc/0x140 | el0t_64_sync+0x18c/0x190 | ---[ end trace 0000000000000000 ]--- We can solve this by consistently determining whether to use a PLT entry for an address. Note that since (the earlier) commit:f1a54ae9af
("arm64: module/ftrace: intialize PLT at load time") ... we can consistently determine the PLT address that a given callsite will use, and therefore ftrace_make_nop() does not need to skip validation when a PLT is in use. This patch factors the existing logic out of ftrace_make_call() and ftrace_make_nop() into a common ftrace_find_callable_addr() helper function, which is used by ftrace_make_call(), ftrace_make_nop(), and ftrace_modify_call(). In ftrace_make_nop() the patching is consistently validated by ftrace_modify_code() as we can always determine what the old instruction should have been. Fixes:3b23e4991f
("arm64: implement ftrace with regs") Signed-off-by: Mark Rutland <mark.rutland@arm.com> Cc: Ard Biesheuvel <ardb@kernel.org> Cc: Will Deacon <will@kernel.org> Tested-by: "Ivan T. Ivanov" <iivanov@suse.de> Reviewed-by: Chengming Zhou <zhouchengming@bytedance.com> Reviewed-by: Ard Biesheuvel <ardb@kernel.org> Link: https://lore.kernel.org/r/20220614080944.1349146-3-mark.rutland@arm.com Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
314 lines
8.5 KiB
C
314 lines
8.5 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* arch/arm64/kernel/ftrace.c
|
|
*
|
|
* Copyright (C) 2013 Linaro Limited
|
|
* Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
|
|
*/
|
|
|
|
#include <linux/ftrace.h>
|
|
#include <linux/module.h>
|
|
#include <linux/swab.h>
|
|
#include <linux/uaccess.h>
|
|
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/debug-monitors.h>
|
|
#include <asm/ftrace.h>
|
|
#include <asm/insn.h>
|
|
#include <asm/patching.h>
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
|
/*
|
|
* Replace a single instruction, which may be a branch or NOP.
|
|
* If @validate == true, a replaced instruction is checked against 'old'.
|
|
*/
|
|
static int ftrace_modify_code(unsigned long pc, u32 old, u32 new,
|
|
bool validate)
|
|
{
|
|
u32 replaced;
|
|
|
|
/*
|
|
* Note:
|
|
* We are paranoid about modifying text, as if a bug were to happen, it
|
|
* could cause us to read or write to someplace that could cause harm.
|
|
* Carefully read and modify the code with aarch64_insn_*() which uses
|
|
* probe_kernel_*(), and make sure what we read is what we expected it
|
|
* to be before modifying it.
|
|
*/
|
|
if (validate) {
|
|
if (aarch64_insn_read((void *)pc, &replaced))
|
|
return -EFAULT;
|
|
|
|
if (replaced != old)
|
|
return -EINVAL;
|
|
}
|
|
if (aarch64_insn_patch_text_nosync((void *)pc, new))
|
|
return -EPERM;
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Replace tracer function in ftrace_caller()
|
|
*/
|
|
int ftrace_update_ftrace_func(ftrace_func_t func)
|
|
{
|
|
unsigned long pc;
|
|
u32 new;
|
|
|
|
pc = (unsigned long)function_nocfi(ftrace_call);
|
|
new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func,
|
|
AARCH64_INSN_BRANCH_LINK);
|
|
|
|
return ftrace_modify_code(pc, 0, new, false);
|
|
}
|
|
|
|
static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr)
|
|
{
|
|
#ifdef CONFIG_ARM64_MODULE_PLTS
|
|
struct plt_entry *plt = mod->arch.ftrace_trampolines;
|
|
|
|
if (addr == FTRACE_ADDR)
|
|
return &plt[FTRACE_PLT_IDX];
|
|
if (addr == FTRACE_REGS_ADDR &&
|
|
IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
|
|
return &plt[FTRACE_REGS_PLT_IDX];
|
|
#endif
|
|
return NULL;
|
|
}
|
|
|
|
/*
|
|
* Find the address the callsite must branch to in order to reach '*addr'.
|
|
*
|
|
* Due to the limited range of 'BL' instructions, modules may be placed too far
|
|
* away to branch directly and must use a PLT.
|
|
*
|
|
* Returns true when '*addr' contains a reachable target address, or has been
|
|
* modified to contain a PLT address. Returns false otherwise.
|
|
*/
|
|
static bool ftrace_find_callable_addr(struct dyn_ftrace *rec,
|
|
struct module *mod,
|
|
unsigned long *addr)
|
|
{
|
|
unsigned long pc = rec->ip;
|
|
long offset = (long)*addr - (long)pc;
|
|
struct plt_entry *plt;
|
|
|
|
/*
|
|
* When the target is within range of the 'BL' instruction, use 'addr'
|
|
* as-is and branch to that directly.
|
|
*/
|
|
if (offset >= -SZ_128M && offset < SZ_128M)
|
|
return true;
|
|
|
|
/*
|
|
* When the target is outside of the range of a 'BL' instruction, we
|
|
* must use a PLT to reach it. We can only place PLTs for modules, and
|
|
* only when module PLT support is built-in.
|
|
*/
|
|
if (!IS_ENABLED(CONFIG_ARM64_MODULE_PLTS))
|
|
return false;
|
|
|
|
/*
|
|
* 'mod' is only set at module load time, but if we end up
|
|
* dealing with an out-of-range condition, we can assume it
|
|
* is due to a module being loaded far away from the kernel.
|
|
*
|
|
* NOTE: __module_text_address() must be called with preemption
|
|
* disabled, but we can rely on ftrace_lock to ensure that 'mod'
|
|
* retains its validity throughout the remainder of this code.
|
|
*/
|
|
if (!mod) {
|
|
preempt_disable();
|
|
mod = __module_text_address(pc);
|
|
preempt_enable();
|
|
}
|
|
|
|
if (WARN_ON(!mod))
|
|
return false;
|
|
|
|
plt = get_ftrace_plt(mod, *addr);
|
|
if (!plt) {
|
|
pr_err("ftrace: no module PLT for %ps\n", (void *)*addr);
|
|
return false;
|
|
}
|
|
|
|
*addr = (unsigned long)plt;
|
|
return true;
|
|
}
|
|
|
|
/*
|
|
* Turn on the call to ftrace_caller() in instrumented function
|
|
*/
|
|
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
|
|
{
|
|
unsigned long pc = rec->ip;
|
|
u32 old, new;
|
|
|
|
if (!ftrace_find_callable_addr(rec, NULL, &addr))
|
|
return -EINVAL;
|
|
|
|
old = aarch64_insn_gen_nop();
|
|
new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
|
|
|
|
return ftrace_modify_code(pc, old, new, true);
|
|
}
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
|
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
|
|
unsigned long addr)
|
|
{
|
|
unsigned long pc = rec->ip;
|
|
u32 old, new;
|
|
|
|
if (!ftrace_find_callable_addr(rec, NULL, &old_addr))
|
|
return -EINVAL;
|
|
if (!ftrace_find_callable_addr(rec, NULL, &addr))
|
|
return -EINVAL;
|
|
|
|
old = aarch64_insn_gen_branch_imm(pc, old_addr,
|
|
AARCH64_INSN_BRANCH_LINK);
|
|
new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
|
|
|
|
return ftrace_modify_code(pc, old, new, true);
|
|
}
|
|
|
|
/*
|
|
* The compiler has inserted two NOPs before the regular function prologue.
|
|
* All instrumented functions follow the AAPCS, so x0-x8 and x19-x30 are live,
|
|
* and x9-x18 are free for our use.
|
|
*
|
|
* At runtime we want to be able to swing a single NOP <-> BL to enable or
|
|
* disable the ftrace call. The BL requires us to save the original LR value,
|
|
* so here we insert a <MOV X9, LR> over the first NOP so the instructions
|
|
* before the regular prologue are:
|
|
*
|
|
* | Compiled | Disabled | Enabled |
|
|
* +----------+------------+------------+
|
|
* | NOP | MOV X9, LR | MOV X9, LR |
|
|
* | NOP | NOP | BL <entry> |
|
|
*
|
|
* The LR value will be recovered by ftrace_regs_entry, and restored into LR
|
|
* before returning to the regular function prologue. When a function is not
|
|
* being traced, the MOV is not harmful given x9 is not live per the AAPCS.
|
|
*
|
|
* Note: ftrace_process_locs() has pre-adjusted rec->ip to be the address of
|
|
* the BL.
|
|
*/
|
|
int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
|
|
{
|
|
unsigned long pc = rec->ip - AARCH64_INSN_SIZE;
|
|
u32 old, new;
|
|
|
|
old = aarch64_insn_gen_nop();
|
|
new = aarch64_insn_gen_move_reg(AARCH64_INSN_REG_9,
|
|
AARCH64_INSN_REG_LR,
|
|
AARCH64_INSN_VARIANT_64BIT);
|
|
return ftrace_modify_code(pc, old, new, true);
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* Turn off the call to ftrace_caller() in instrumented function
|
|
*/
|
|
int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
|
|
unsigned long addr)
|
|
{
|
|
unsigned long pc = rec->ip;
|
|
u32 old = 0, new;
|
|
|
|
if (!ftrace_find_callable_addr(rec, mod, &addr))
|
|
return -EINVAL;
|
|
|
|
old = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
|
|
new = aarch64_insn_gen_nop();
|
|
|
|
return ftrace_modify_code(pc, old, new, true);
|
|
}
|
|
|
|
void arch_ftrace_update_code(int command)
|
|
{
|
|
command |= FTRACE_MAY_SLEEP;
|
|
ftrace_modify_all_code(command);
|
|
}
|
|
#endif /* CONFIG_DYNAMIC_FTRACE */
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
/*
|
|
* function_graph tracer expects ftrace_return_to_handler() to be called
|
|
* on the way back to parent. For this purpose, this function is called
|
|
* in _mcount() or ftrace_caller() to replace return address (*parent) on
|
|
* the call stack to return_to_handler.
|
|
*/
|
|
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
|
|
unsigned long frame_pointer)
|
|
{
|
|
unsigned long return_hooker = (unsigned long)&return_to_handler;
|
|
unsigned long old;
|
|
|
|
if (unlikely(atomic_read(¤t->tracing_graph_pause)))
|
|
return;
|
|
|
|
/*
|
|
* Note:
|
|
* No protection against faulting at *parent, which may be seen
|
|
* on other archs. It's unlikely on AArch64.
|
|
*/
|
|
old = *parent;
|
|
|
|
if (!function_graph_enter(old, self_addr, frame_pointer,
|
|
(void *)frame_pointer)) {
|
|
*parent = return_hooker;
|
|
}
|
|
}
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
|
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
|
|
struct ftrace_ops *op, struct ftrace_regs *fregs)
|
|
{
|
|
/*
|
|
* When DYNAMIC_FTRACE_WITH_REGS is selected, `fregs` can never be NULL
|
|
* and arch_ftrace_get_regs(fregs) will always give a non-NULL pt_regs
|
|
* in which we can safely modify the LR.
|
|
*/
|
|
struct pt_regs *regs = arch_ftrace_get_regs(fregs);
|
|
unsigned long *parent = (unsigned long *)&procedure_link_pointer(regs);
|
|
|
|
prepare_ftrace_return(ip, parent, frame_pointer(regs));
|
|
}
|
|
#else
|
|
/*
|
|
* Turn on/off the call to ftrace_graph_caller() in ftrace_caller()
|
|
* depending on @enable.
|
|
*/
|
|
static int ftrace_modify_graph_caller(bool enable)
|
|
{
|
|
unsigned long pc = (unsigned long)&ftrace_graph_call;
|
|
u32 branch, nop;
|
|
|
|
branch = aarch64_insn_gen_branch_imm(pc,
|
|
(unsigned long)ftrace_graph_caller,
|
|
AARCH64_INSN_BRANCH_NOLINK);
|
|
nop = aarch64_insn_gen_nop();
|
|
|
|
if (enable)
|
|
return ftrace_modify_code(pc, nop, branch, true);
|
|
else
|
|
return ftrace_modify_code(pc, branch, nop, true);
|
|
}
|
|
|
|
int ftrace_enable_ftrace_graph_caller(void)
|
|
{
|
|
return ftrace_modify_graph_caller(true);
|
|
}
|
|
|
|
int ftrace_disable_ftrace_graph_caller(void)
|
|
{
|
|
return ftrace_modify_graph_caller(false);
|
|
}
|
|
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
|
|
#endif /* CONFIG_DYNAMIC_FTRACE */
|
|
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
|