mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-11-26 13:44:15 +08:00
Merge branch 'for-next/insn' into for-next/core
Refactoring of our instruction decoding routines and addition of some missing encodings. * for-next/insn: arm64: insn: avoid circular include dependency arm64: insn: move AARCH64_INSN_SIZE into <asm/insn.h> arm64: insn: decouple patching from insn code arm64: insn: Add load/store decoding helpers arm64: insn: Add some opcodes to instruction decoder arm64: insn: Add barrier encodings arm64: insn: Add SVE instruction class arm64: Move instruction encoder/decoder under lib/ arm64: Move aarch32 condition check functions arm64: Move patching utilities out of instruction encoding/decoding
This commit is contained in:
Will Deacon
commit
181a126979
@ -3,12 +3,10 @@
|
||||
#define __ASM_ALTERNATIVE_MACROS_H
|
||||
|
||||
#include <asm/cpucaps.h>
|
||||
#include <asm/insn-def.h>
|
||||
|
||||
#define ARM64_CB_PATCH ARM64_NCAPS
|
||||
|
||||
/* A64 instructions are always 32 bits. */
|
||||
#define AARCH64_INSN_SIZE 4
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
#include <linux/stringify.h>
|
||||
|
||||
9
arch/arm64/include/asm/insn-def.h
Normal file
9
arch/arm64/include/asm/insn-def.h
Normal file
@ -0,0 +1,9 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
|
||||
#ifndef __ASM_INSN_DEF_H
|
||||
#define __ASM_INSN_DEF_H
|
||||
|
||||
/* A64 instructions are always 32 bits. */
|
||||
#define AARCH64_INSN_SIZE 4
|
||||
|
||||
#endif /* __ASM_INSN_DEF_H */
|
||||
@ -10,7 +10,7 @@
|
||||
#include <linux/build_bug.h>
|
||||
#include <linux/types.h>
|
||||
|
||||
#include <asm/alternative.h>
|
||||
#include <asm/insn-def.h>
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
/*
|
||||
@ -30,6 +30,7 @@
|
||||
*/
|
||||
enum aarch64_insn_encoding_class {
|
||||
AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */
|
||||
AARCH64_INSN_CLS_SVE, /* SVE instructions */
|
||||
AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */
|
||||
AARCH64_INSN_CLS_DP_REG, /* Data processing - register */
|
||||
AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */
|
||||
@ -294,6 +295,12 @@ __AARCH64_INSN_FUNCS(adr, 0x9F000000, 0x10000000)
|
||||
__AARCH64_INSN_FUNCS(adrp, 0x9F000000, 0x90000000)
|
||||
__AARCH64_INSN_FUNCS(prfm, 0x3FC00000, 0x39800000)
|
||||
__AARCH64_INSN_FUNCS(prfm_lit, 0xFF000000, 0xD8000000)
|
||||
__AARCH64_INSN_FUNCS(store_imm, 0x3FC00000, 0x39000000)
|
||||
__AARCH64_INSN_FUNCS(load_imm, 0x3FC00000, 0x39400000)
|
||||
__AARCH64_INSN_FUNCS(store_pre, 0x3FE00C00, 0x38000C00)
|
||||
__AARCH64_INSN_FUNCS(load_pre, 0x3FE00C00, 0x38400C00)
|
||||
__AARCH64_INSN_FUNCS(store_post, 0x3FE00C00, 0x38000400)
|
||||
__AARCH64_INSN_FUNCS(load_post, 0x3FE00C00, 0x38400400)
|
||||
__AARCH64_INSN_FUNCS(str_reg, 0x3FE0EC00, 0x38206800)
|
||||
__AARCH64_INSN_FUNCS(ldadd, 0x3F20FC00, 0x38200000)
|
||||
__AARCH64_INSN_FUNCS(ldr_reg, 0x3FE0EC00, 0x38606800)
|
||||
@ -302,6 +309,8 @@ __AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000)
|
||||
__AARCH64_INSN_FUNCS(exclusive, 0x3F800000, 0x08000000)
|
||||
__AARCH64_INSN_FUNCS(load_ex, 0x3F400000, 0x08400000)
|
||||
__AARCH64_INSN_FUNCS(store_ex, 0x3F400000, 0x08000000)
|
||||
__AARCH64_INSN_FUNCS(stp, 0x7FC00000, 0x29000000)
|
||||
__AARCH64_INSN_FUNCS(ldp, 0x7FC00000, 0x29400000)
|
||||
__AARCH64_INSN_FUNCS(stp_post, 0x7FC00000, 0x28800000)
|
||||
__AARCH64_INSN_FUNCS(ldp_post, 0x7FC00000, 0x28C00000)
|
||||
__AARCH64_INSN_FUNCS(stp_pre, 0x7FC00000, 0x29800000)
|
||||
@ -334,6 +343,7 @@ __AARCH64_INSN_FUNCS(rev64, 0x7FFFFC00, 0x5AC00C00)
|
||||
__AARCH64_INSN_FUNCS(and, 0x7F200000, 0x0A000000)
|
||||
__AARCH64_INSN_FUNCS(bic, 0x7F200000, 0x0A200000)
|
||||
__AARCH64_INSN_FUNCS(orr, 0x7F200000, 0x2A000000)
|
||||
__AARCH64_INSN_FUNCS(mov_reg, 0x7FE0FFE0, 0x2A0003E0)
|
||||
__AARCH64_INSN_FUNCS(orn, 0x7F200000, 0x2A200000)
|
||||
__AARCH64_INSN_FUNCS(eor, 0x7F200000, 0x4A000000)
|
||||
__AARCH64_INSN_FUNCS(eon, 0x7F200000, 0x4A200000)
|
||||
@ -368,6 +378,14 @@ __AARCH64_INSN_FUNCS(eret_auth, 0xFFFFFBFF, 0xD69F0BFF)
|
||||
__AARCH64_INSN_FUNCS(mrs, 0xFFF00000, 0xD5300000)
|
||||
__AARCH64_INSN_FUNCS(msr_imm, 0xFFF8F01F, 0xD500401F)
|
||||
__AARCH64_INSN_FUNCS(msr_reg, 0xFFF00000, 0xD5100000)
|
||||
__AARCH64_INSN_FUNCS(dmb, 0xFFFFF0FF, 0xD50330BF)
|
||||
__AARCH64_INSN_FUNCS(dsb_base, 0xFFFFF0FF, 0xD503309F)
|
||||
__AARCH64_INSN_FUNCS(dsb_nxs, 0xFFFFF3FF, 0xD503323F)
|
||||
__AARCH64_INSN_FUNCS(isb, 0xFFFFF0FF, 0xD50330DF)
|
||||
__AARCH64_INSN_FUNCS(sb, 0xFFFFFFFF, 0xD50330FF)
|
||||
__AARCH64_INSN_FUNCS(clrex, 0xFFFFF0FF, 0xD503305F)
|
||||
__AARCH64_INSN_FUNCS(ssbb, 0xFFFFFFFF, 0xD503309F)
|
||||
__AARCH64_INSN_FUNCS(pssbb, 0xFFFFFFFF, 0xD503349F)
|
||||
|
||||
#undef __AARCH64_INSN_FUNCS
|
||||
|
||||
@ -379,8 +397,47 @@ static inline bool aarch64_insn_is_adr_adrp(u32 insn)
|
||||
return aarch64_insn_is_adr(insn) || aarch64_insn_is_adrp(insn);
|
||||
}
|
||||
|
||||
int aarch64_insn_read(void *addr, u32 *insnp);
|
||||
int aarch64_insn_write(void *addr, u32 insn);
|
||||
static inline bool aarch64_insn_is_dsb(u32 insn)
|
||||
{
|
||||
return aarch64_insn_is_dsb_base(insn) || aarch64_insn_is_dsb_nxs(insn);
|
||||
}
|
||||
|
||||
static inline bool aarch64_insn_is_barrier(u32 insn)
|
||||
{
|
||||
return aarch64_insn_is_dmb(insn) || aarch64_insn_is_dsb(insn) ||
|
||||
aarch64_insn_is_isb(insn) || aarch64_insn_is_sb(insn) ||
|
||||
aarch64_insn_is_clrex(insn) || aarch64_insn_is_ssbb(insn) ||
|
||||
aarch64_insn_is_pssbb(insn);
|
||||
}
|
||||
|
||||
static inline bool aarch64_insn_is_store_single(u32 insn)
|
||||
{
|
||||
return aarch64_insn_is_store_imm(insn) ||
|
||||
aarch64_insn_is_store_pre(insn) ||
|
||||
aarch64_insn_is_store_post(insn);
|
||||
}
|
||||
|
||||
static inline bool aarch64_insn_is_store_pair(u32 insn)
|
||||
{
|
||||
return aarch64_insn_is_stp(insn) ||
|
||||
aarch64_insn_is_stp_pre(insn) ||
|
||||
aarch64_insn_is_stp_post(insn);
|
||||
}
|
||||
|
||||
static inline bool aarch64_insn_is_load_single(u32 insn)
|
||||
{
|
||||
return aarch64_insn_is_load_imm(insn) ||
|
||||
aarch64_insn_is_load_pre(insn) ||
|
||||
aarch64_insn_is_load_post(insn);
|
||||
}
|
||||
|
||||
static inline bool aarch64_insn_is_load_pair(u32 insn)
|
||||
{
|
||||
return aarch64_insn_is_ldp(insn) ||
|
||||
aarch64_insn_is_ldp_pre(insn) ||
|
||||
aarch64_insn_is_ldp_post(insn);
|
||||
}
|
||||
|
||||
enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
|
||||
bool aarch64_insn_uses_literal(u32 insn);
|
||||
bool aarch64_insn_is_branch(u32 insn);
|
||||
@ -487,9 +544,6 @@ u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
|
||||
s32 aarch64_get_branch_offset(u32 insn);
|
||||
u32 aarch64_set_branch_offset(u32 insn, s32 offset);
|
||||
|
||||
int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
|
||||
int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
|
||||
|
||||
s32 aarch64_insn_adrp_get_offset(u32 insn);
|
||||
u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset);
|
||||
|
||||
@ -506,6 +560,7 @@ u32 aarch32_insn_mcr_extract_crm(u32 insn);
|
||||
|
||||
typedef bool (pstate_check_t)(unsigned long);
|
||||
extern pstate_check_t * const aarch32_opcode_cond_checks[16];
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#endif /* __ASM_INSN_H */
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
#define __ARM_KVM_ASM_H__
|
||||
|
||||
#include <asm/hyp_image.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/virt.h>
|
||||
|
||||
#define ARM_EXIT_WITH_SERROR_BIT 31
|
||||
|
||||
13
arch/arm64/include/asm/patching.h
Normal file
13
arch/arm64/include/asm/patching.h
Normal file
@ -0,0 +1,13 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
#ifndef __ASM_PATCHING_H
|
||||
#define __ASM_PATCHING_H
|
||||
|
||||
#include <linux/types.h>
|
||||
|
||||
int aarch64_insn_read(void *addr, u32 *insnp);
|
||||
int aarch64_insn_write(void *addr, u32 insn);
|
||||
|
||||
int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
|
||||
int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
|
||||
|
||||
#endif /* __ASM_PATCHING_H */
|
||||
@ -24,11 +24,12 @@ KCOV_INSTRUMENT_idle.o := n
|
||||
obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
|
||||
entry-common.o entry-fpsimd.o process.o ptrace.o \
|
||||
setup.o signal.o sys.o stacktrace.o time.o traps.o \
|
||||
io.o vdso.o hyp-stub.o psci.o cpu_ops.o insn.o \
|
||||
io.o vdso.o hyp-stub.o psci.o cpu_ops.o \
|
||||
return_address.o cpuinfo.o cpu_errata.o \
|
||||
cpufeature.o alternative.o cacheinfo.o \
|
||||
smp.o smp_spin_table.o topology.o smccc-call.o \
|
||||
syscall.o proton-pack.o idreg-override.o idle.o
|
||||
syscall.o proton-pack.o idreg-override.o idle.o \
|
||||
patching.o
|
||||
|
||||
targets += efi-entry.o
|
||||
|
||||
|
||||
@ -76,6 +76,7 @@
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/cpu_ops.h>
|
||||
#include <asm/fpsimd.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/kvm_host.h>
|
||||
#include <asm/mmu_context.h>
|
||||
#include <asm/mte.h>
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/ftrace.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/patching.h>
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE
|
||||
/*
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/jump_label.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/patching.h>
|
||||
|
||||
void arch_jump_label_transform(struct jump_entry *entry,
|
||||
enum jump_label_type type)
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/patching.h>
|
||||
#include <asm/traps.h>
|
||||
|
||||
struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = {
|
||||
|
||||
150
arch/arm64/kernel/patching.c
Normal file
150
arch/arm64/kernel/patching.c
Normal file
@ -0,0 +1,150 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/smp.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/stop_machine.h>
|
||||
#include <linux/uaccess.h>
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/fixmap.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/kprobes.h>
|
||||
#include <asm/patching.h>
|
||||
#include <asm/sections.h>
|
||||
|
||||
static DEFINE_RAW_SPINLOCK(patch_lock);
|
||||
|
||||
static bool is_exit_text(unsigned long addr)
|
||||
{
|
||||
/* discarded with init text/data */
|
||||
return system_state < SYSTEM_RUNNING &&
|
||||
addr >= (unsigned long)__exittext_begin &&
|
||||
addr < (unsigned long)__exittext_end;
|
||||
}
|
||||
|
||||
static bool is_image_text(unsigned long addr)
|
||||
{
|
||||
return core_kernel_text(addr) || is_exit_text(addr);
|
||||
}
|
||||
|
||||
static void __kprobes *patch_map(void *addr, int fixmap)
|
||||
{
|
||||
unsigned long uintaddr = (uintptr_t) addr;
|
||||
bool image = is_image_text(uintaddr);
|
||||
struct page *page;
|
||||
|
||||
if (image)
|
||||
page = phys_to_page(__pa_symbol(addr));
|
||||
else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
|
||||
page = vmalloc_to_page(addr);
|
||||
else
|
||||
return addr;
|
||||
|
||||
BUG_ON(!page);
|
||||
return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
|
||||
(uintaddr & ~PAGE_MASK));
|
||||
}
|
||||
|
||||
static void __kprobes patch_unmap(int fixmap)
|
||||
{
|
||||
clear_fixmap(fixmap);
|
||||
}
|
||||
/*
|
||||
* In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
|
||||
* little-endian.
|
||||
*/
|
||||
int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
|
||||
{
|
||||
int ret;
|
||||
__le32 val;
|
||||
|
||||
ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE);
|
||||
if (!ret)
|
||||
*insnp = le32_to_cpu(val);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
|
||||
{
|
||||
void *waddr = addr;
|
||||
unsigned long flags = 0;
|
||||
int ret;
|
||||
|
||||
raw_spin_lock_irqsave(&patch_lock, flags);
|
||||
waddr = patch_map(addr, FIX_TEXT_POKE0);
|
||||
|
||||
ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE);
|
||||
|
||||
patch_unmap(FIX_TEXT_POKE0);
|
||||
raw_spin_unlock_irqrestore(&patch_lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int __kprobes aarch64_insn_write(void *addr, u32 insn)
|
||||
{
|
||||
return __aarch64_insn_write(addr, cpu_to_le32(insn));
|
||||
}
|
||||
|
||||
int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
|
||||
{
|
||||
u32 *tp = addr;
|
||||
int ret;
|
||||
|
||||
/* A64 instructions must be word aligned */
|
||||
if ((uintptr_t)tp & 0x3)
|
||||
return -EINVAL;
|
||||
|
||||
ret = aarch64_insn_write(tp, insn);
|
||||
if (ret == 0)
|
||||
caches_clean_inval_pou((uintptr_t)tp,
|
||||
(uintptr_t)tp + AARCH64_INSN_SIZE);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
struct aarch64_insn_patch {
|
||||
void **text_addrs;
|
||||
u32 *new_insns;
|
||||
int insn_cnt;
|
||||
atomic_t cpu_count;
|
||||
};
|
||||
|
||||
static int __kprobes aarch64_insn_patch_text_cb(void *arg)
|
||||
{
|
||||
int i, ret = 0;
|
||||
struct aarch64_insn_patch *pp = arg;
|
||||
|
||||
/* The first CPU becomes master */
|
||||
if (atomic_inc_return(&pp->cpu_count) == 1) {
|
||||
for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
|
||||
ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
|
||||
pp->new_insns[i]);
|
||||
/* Notify other processors with an additional increment. */
|
||||
atomic_inc(&pp->cpu_count);
|
||||
} else {
|
||||
while (atomic_read(&pp->cpu_count) <= num_online_cpus())
|
||||
cpu_relax();
|
||||
isb();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
|
||||
{
|
||||
struct aarch64_insn_patch patch = {
|
||||
.text_addrs = addrs,
|
||||
.new_insns = insns,
|
||||
.insn_cnt = cnt,
|
||||
.cpu_count = ATOMIC_INIT(0),
|
||||
};
|
||||
|
||||
if (cnt <= 0)
|
||||
return -EINVAL;
|
||||
|
||||
return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch,
|
||||
cpu_online_mask);
|
||||
}
|
||||
@ -7,26 +7,28 @@
|
||||
* Copyright (C) 2013 Linaro Limited.
|
||||
* Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org>
|
||||
*/
|
||||
#include <linux/extable.h>
|
||||
#include <linux/kasan.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/kprobes.h>
|
||||
#include <linux/extable.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/stop_machine.h>
|
||||
#include <linux/sched/debug.h>
|
||||
#include <linux/set_memory.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/stop_machine.h>
|
||||
#include <linux/stringify.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/ptrace.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/daifflags.h>
|
||||
#include <asm/system_misc.h>
|
||||
#include <asm/insn.h>
|
||||
#include <linux/uaccess.h>
|
||||
#include <linux/vmalloc.h>
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/daifflags.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/irq.h>
|
||||
#include <asm/patching.h>
|
||||
#include <asm/ptrace.h>
|
||||
#include <asm/sections.h>
|
||||
#include <asm/system_misc.h>
|
||||
#include <asm/traps.h>
|
||||
|
||||
#include "decode-insn.h"
|
||||
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
#include <linux/kprobes.h>
|
||||
|
||||
#include <asm/ptrace.h>
|
||||
#include <asm/traps.h>
|
||||
|
||||
#include "simulate-insn.h"
|
||||
|
||||
|
||||
@ -38,6 +38,7 @@
|
||||
#include <asm/extable.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/kprobes.h>
|
||||
#include <asm/patching.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/smp.h>
|
||||
#include <asm/stack_pointer.h>
|
||||
@ -45,6 +46,104 @@
|
||||
#include <asm/system_misc.h>
|
||||
#include <asm/sysreg.h>
|
||||
|
||||
static bool __kprobes __check_eq(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_Z_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_ne(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_Z_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_cs(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_C_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_cc(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_C_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_mi(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_N_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_pl(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_N_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_vs(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_V_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_vc(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_V_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_hi(unsigned long pstate)
|
||||
{
|
||||
pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
|
||||
return (pstate & PSR_C_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_ls(unsigned long pstate)
|
||||
{
|
||||
pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
|
||||
return (pstate & PSR_C_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_ge(unsigned long pstate)
|
||||
{
|
||||
pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
|
||||
return (pstate & PSR_N_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_lt(unsigned long pstate)
|
||||
{
|
||||
pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
|
||||
return (pstate & PSR_N_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_gt(unsigned long pstate)
|
||||
{
|
||||
/*PSR_N_BIT ^= PSR_V_BIT */
|
||||
unsigned long temp = pstate ^ (pstate << 3);
|
||||
|
||||
temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
|
||||
return (temp & PSR_N_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_le(unsigned long pstate)
|
||||
{
|
||||
/*PSR_N_BIT ^= PSR_V_BIT */
|
||||
unsigned long temp = pstate ^ (pstate << 3);
|
||||
|
||||
temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
|
||||
return (temp & PSR_N_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_al(unsigned long pstate)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that
|
||||
* it behaves identically to 0b1110 ("al").
|
||||
*/
|
||||
pstate_check_t * const aarch32_opcode_cond_checks[16] = {
|
||||
__check_eq, __check_ne, __check_cs, __check_cc,
|
||||
__check_mi, __check_pl, __check_vs, __check_vc,
|
||||
__check_hi, __check_ls, __check_ge, __check_lt,
|
||||
__check_gt, __check_le, __check_al, __check_al
|
||||
};
|
||||
|
||||
int show_unhandled_signals = 0;
|
||||
|
||||
static void dump_kernel_instr(const char *lvl, struct pt_regs *regs)
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
lib-y := clear_user.o delay.o copy_from_user.o \
|
||||
copy_to_user.o copy_in_user.o copy_page.o \
|
||||
clear_page.o csum.o memchr.o memcpy.o \
|
||||
clear_page.o csum.o insn.o memchr.o memcpy.o \
|
||||
memset.o memcmp.o strcmp.o strncmp.o strlen.o \
|
||||
strnlen.o strchr.o strrchr.o tishift.o
|
||||
|
||||
|
||||
@ -7,21 +7,14 @@
|
||||
*/
|
||||
#include <linux/bitops.h>
|
||||
#include <linux/bug.h>
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/smp.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/stop_machine.h>
|
||||
#include <linux/printk.h>
|
||||
#include <linux/sizes.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/uaccess.h>
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/fixmap.h>
|
||||
#include <asm/errno.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/kprobes.h>
|
||||
#include <asm/sections.h>
|
||||
|
||||
#define AARCH64_INSN_SF_BIT BIT(31)
|
||||
#define AARCH64_INSN_N_BIT BIT(22)
|
||||
@ -30,7 +23,7 @@
|
||||
static const int aarch64_insn_encoding_class[] = {
|
||||
AARCH64_INSN_CLS_UNKNOWN,
|
||||
AARCH64_INSN_CLS_UNKNOWN,
|
||||
AARCH64_INSN_CLS_UNKNOWN,
|
||||
AARCH64_INSN_CLS_SVE,
|
||||
AARCH64_INSN_CLS_UNKNOWN,
|
||||
AARCH64_INSN_CLS_LDST,
|
||||
AARCH64_INSN_CLS_DP_REG,
|
||||
@ -83,81 +76,6 @@ bool aarch64_insn_is_branch_imm(u32 insn)
|
||||
aarch64_insn_is_bcond(insn));
|
||||
}
|
||||
|
||||
static DEFINE_RAW_SPINLOCK(patch_lock);
|
||||
|
||||
static bool is_exit_text(unsigned long addr)
|
||||
{
|
||||
/* discarded with init text/data */
|
||||
return system_state < SYSTEM_RUNNING &&
|
||||
addr >= (unsigned long)__exittext_begin &&
|
||||
addr < (unsigned long)__exittext_end;
|
||||
}
|
||||
|
||||
static bool is_image_text(unsigned long addr)
|
||||
{
|
||||
return core_kernel_text(addr) || is_exit_text(addr);
|
||||
}
|
||||
|
||||
static void __kprobes *patch_map(void *addr, int fixmap)
|
||||
{
|
||||
unsigned long uintaddr = (uintptr_t) addr;
|
||||
bool image = is_image_text(uintaddr);
|
||||
struct page *page;
|
||||
|
||||
if (image)
|
||||
page = phys_to_page(__pa_symbol(addr));
|
||||
else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
|
||||
page = vmalloc_to_page(addr);
|
||||
else
|
||||
return addr;
|
||||
|
||||
BUG_ON(!page);
|
||||
return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
|
||||
(uintaddr & ~PAGE_MASK));
|
||||
}
|
||||
|
||||
static void __kprobes patch_unmap(int fixmap)
|
||||
{
|
||||
clear_fixmap(fixmap);
|
||||
}
|
||||
/*
|
||||
* In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
|
||||
* little-endian.
|
||||
*/
|
||||
int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
|
||||
{
|
||||
int ret;
|
||||
__le32 val;
|
||||
|
||||
ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE);
|
||||
if (!ret)
|
||||
*insnp = le32_to_cpu(val);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
|
||||
{
|
||||
void *waddr = addr;
|
||||
unsigned long flags = 0;
|
||||
int ret;
|
||||
|
||||
raw_spin_lock_irqsave(&patch_lock, flags);
|
||||
waddr = patch_map(addr, FIX_TEXT_POKE0);
|
||||
|
||||
ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE);
|
||||
|
||||
patch_unmap(FIX_TEXT_POKE0);
|
||||
raw_spin_unlock_irqrestore(&patch_lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int __kprobes aarch64_insn_write(void *addr, u32 insn)
|
||||
{
|
||||
return __aarch64_insn_write(addr, cpu_to_le32(insn));
|
||||
}
|
||||
|
||||
bool __kprobes aarch64_insn_uses_literal(u32 insn)
|
||||
{
|
||||
/* ldr/ldrsw (literal), prfm */
|
||||
@ -187,67 +105,6 @@ bool __kprobes aarch64_insn_is_branch(u32 insn)
|
||||
aarch64_insn_is_bcond(insn);
|
||||
}
|
||||
|
||||
int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
|
||||
{
|
||||
u32 *tp = addr;
|
||||
int ret;
|
||||
|
||||
/* A64 instructions must be word aligned */
|
||||
if ((uintptr_t)tp & 0x3)
|
||||
return -EINVAL;
|
||||
|
||||
ret = aarch64_insn_write(tp, insn);
|
||||
if (ret == 0)
|
||||
caches_clean_inval_pou((uintptr_t)tp,
|
||||
(uintptr_t)tp + AARCH64_INSN_SIZE);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
struct aarch64_insn_patch {
|
||||
void **text_addrs;
|
||||
u32 *new_insns;
|
||||
int insn_cnt;
|
||||
atomic_t cpu_count;
|
||||
};
|
||||
|
||||
static int __kprobes aarch64_insn_patch_text_cb(void *arg)
|
||||
{
|
||||
int i, ret = 0;
|
||||
struct aarch64_insn_patch *pp = arg;
|
||||
|
||||
/* The first CPU becomes master */
|
||||
if (atomic_inc_return(&pp->cpu_count) == 1) {
|
||||
for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
|
||||
ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
|
||||
pp->new_insns[i]);
|
||||
/* Notify other processors with an additional increment. */
|
||||
atomic_inc(&pp->cpu_count);
|
||||
} else {
|
||||
while (atomic_read(&pp->cpu_count) <= num_online_cpus())
|
||||
cpu_relax();
|
||||
isb();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
|
||||
{
|
||||
struct aarch64_insn_patch patch = {
|
||||
.text_addrs = addrs,
|
||||
.new_insns = insns,
|
||||
.insn_cnt = cnt,
|
||||
.cpu_count = ATOMIC_INIT(0),
|
||||
};
|
||||
|
||||
if (cnt <= 0)
|
||||
return -EINVAL;
|
||||
|
||||
return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch,
|
||||
cpu_online_mask);
|
||||
}
|
||||
|
||||
static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
|
||||
u32 *maskp, int *shiftp)
|
||||
{
|
||||
@ -1432,104 +1289,6 @@ u32 aarch32_insn_mcr_extract_crm(u32 insn)
|
||||
return insn & CRM_MASK;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_eq(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_Z_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_ne(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_Z_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_cs(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_C_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_cc(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_C_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_mi(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_N_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_pl(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_N_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_vs(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_V_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_vc(unsigned long pstate)
|
||||
{
|
||||
return (pstate & PSR_V_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_hi(unsigned long pstate)
|
||||
{
|
||||
pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
|
||||
return (pstate & PSR_C_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_ls(unsigned long pstate)
|
||||
{
|
||||
pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
|
||||
return (pstate & PSR_C_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_ge(unsigned long pstate)
|
||||
{
|
||||
pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
|
||||
return (pstate & PSR_N_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_lt(unsigned long pstate)
|
||||
{
|
||||
pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
|
||||
return (pstate & PSR_N_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_gt(unsigned long pstate)
|
||||
{
|
||||
/*PSR_N_BIT ^= PSR_V_BIT */
|
||||
unsigned long temp = pstate ^ (pstate << 3);
|
||||
|
||||
temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
|
||||
return (temp & PSR_N_BIT) == 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_le(unsigned long pstate)
|
||||
{
|
||||
/*PSR_N_BIT ^= PSR_V_BIT */
|
||||
unsigned long temp = pstate ^ (pstate << 3);
|
||||
|
||||
temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
|
||||
return (temp & PSR_N_BIT) != 0;
|
||||
}
|
||||
|
||||
static bool __kprobes __check_al(unsigned long pstate)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that
|
||||
* it behaves identically to 0b1110 ("al").
|
||||
*/
|
||||
pstate_check_t * const aarch32_opcode_cond_checks[16] = {
|
||||
__check_eq, __check_ne, __check_cs, __check_cc,
|
||||
__check_mi, __check_pl, __check_vs, __check_vc,
|
||||
__check_hi, __check_ls, __check_ge, __check_lt,
|
||||
__check_gt, __check_le, __check_al, __check_al
|
||||
};
|
||||
|
||||
static bool range_of_ones(u64 val)
|
||||
{
|
||||
/* Doesn't handle full ones or full zeroes */
|
||||
@ -16,6 +16,7 @@
|
||||
#include <asm/byteorder.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/set_memory.h>
|
||||
|
||||
#include "bpf_jit.h"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user