korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-11-26 05:34:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'x86/asm' into locking/core

Browse Source 
We need the ASM_UNREACHABLE() macro for a dependent patch.

Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
Ingo Molnar 2017-08-18 10:29:54 +02:00
commit 0c23647913
32 changed files with 1618 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/x86/Kconfig.debug
View File

@ -305,8 +305,6 @@ config DEBUG_ENTRY
Some of these sanity checks may slow down kernel entries and
exits or otherwise impact performance.
This is currently used to help test NMI code.
If unsure, say N.
config DEBUG_NMI_SELFTEST

1
arch/x86/entry/Makefile
View File

@ -2,7 +2,6 @@
# Makefile for the x86 low level entry code
#
OBJECT_FILES_NON_STANDARD_entry_$(BITS).o := y
OBJECT_FILES_NON_STANDARD_entry_64_compat.o := y
CFLAGS_syscall_64.o += $(call cc-option,-Wno-override-init,)

5
arch/x86/entry/calling.h
View File

@ -1,4 +1,5 @@
#include <linux/jump_label.h>
#include <asm/unwind_hints.h>
/*
@ -112,6 +113,7 @@ For 32-bit we have the following conventions - kernel is built with
movq %rdx, 12*8+\offset(%rsp)
movq %rsi, 13*8+\offset(%rsp)
movq %rdi, 14*8+\offset(%rsp)
UNWIND_HINT_REGS offset=\offset extra=0
.endm
.macro SAVE_C_REGS offset=0
SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1
@ -136,6 +138,7 @@ For 32-bit we have the following conventions - kernel is built with
movq %r12, 3*8+\offset(%rsp)
movq %rbp, 4*8+\offset(%rsp)
movq %rbx, 5*8+\offset(%rsp)
UNWIND_HINT_REGS offset=\offset
.endm
.macro RESTORE_EXTRA_REGS offset=0
@ -145,6 +148,7 @@ For 32-bit we have the following conventions - kernel is built with
movq 3*8+\offset(%rsp), %r12
movq 4*8+\offset(%rsp), %rbp
movq 5*8+\offset(%rsp), %rbx
UNWIND_HINT_REGS offset=\offset extra=0
.endm
.macro RESTORE_C_REGS_HELPER rstor_rax=1, rstor_rcx=1, rstor_r11=1, rstor_r8910=1, rstor_rdx=1
@ -167,6 +171,7 @@ For 32-bit we have the following conventions - kernel is built with
.endif
movq 13*8(%rsp), %rsi
movq 14*8(%rsp), %rdi
UNWIND_HINT_IRET_REGS offset=16*8
.endm
.macro RESTORE_C_REGS
RESTORE_C_REGS_HELPER 1,1,1,1,1

170
arch/x86/entry/entry_64.S
View File

@ -36,6 +36,7 @@
#include <asm/smap.h>
#include <asm/pgtable_types.h>
#include <asm/export.h>
#include <asm/frame.h>
#include <linux/err.h>
.code64
@ -43,9 +44,10 @@
#ifdef CONFIG_PARAVIRT
ENTRY(native_usergs_sysret64)
UNWIND_HINT_EMPTY
swapgs
sysretq
ENDPROC(native_usergs_sysret64)
END(native_usergs_sysret64)
#endif /* CONFIG_PARAVIRT */
.macro TRACE_IRQS_IRETQ
@ -134,6 +136,7 @@ ENDPROC(native_usergs_sysret64)
*/
ENTRY(entry_SYSCALL_64)
UNWIND_HINT_EMPTY
/*
* Interrupts are off on entry.
* We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
@ -169,6 +172,7 @@ GLOBAL(entry_SYSCALL_64_after_swapgs)
pushq %r10 /* pt_regs->r10 */
pushq %r11 /* pt_regs->r11 */
sub $(6*8), %rsp /* pt_regs->bp, bx, r12-15 not saved */
UNWIND_HINT_REGS extra=0
/*
* If we need to do entry work or if we guess we'll need to do
@ -223,6 +227,7 @@ entry_SYSCALL_64_fastpath:
movq EFLAGS(%rsp), %r11
RESTORE_C_REGS_EXCEPT_RCX_R11
movq RSP(%rsp), %rsp
UNWIND_HINT_EMPTY
USERGS_SYSRET64
1:
@ -316,6 +321,7 @@ syscall_return_via_sysret:
/* rcx and r11 are already restored (see code above) */
RESTORE_C_REGS_EXCEPT_RCX_R11
movq RSP(%rsp), %rsp
UNWIND_HINT_EMPTY
USERGS_SYSRET64
opportunistic_sysret_failed:
@ -343,6 +349,7 @@ ENTRY(stub_ptregs_64)
DISABLE_INTERRUPTS(CLBR_ANY)
TRACE_IRQS_OFF
popq %rax
UNWIND_HINT_REGS extra=0
jmp entry_SYSCALL64_slow_path
1:
@ -351,6 +358,7 @@ END(stub_ptregs_64)
.macro ptregs_stub func
ENTRY(ptregs_\func)
UNWIND_HINT_FUNC
leaq \func(%rip), %rax
jmp stub_ptregs_64
END(ptregs_\func)
@ -367,6 +375,7 @@ END(ptregs_\func)
* %rsi: next task
*/
ENTRY(__switch_to_asm)
UNWIND_HINT_FUNC
/*
* Save callee-saved registers
* This must match the order in inactive_task_frame
@ -406,6 +415,7 @@ END(__switch_to_asm)
* r12: kernel thread arg
*/
ENTRY(ret_from_fork)
UNWIND_HINT_EMPTY
movq %rax, %rdi
call schedule_tail /* rdi: 'prev' task parameter */
@ -413,6 +423,7 @@ ENTRY(ret_from_fork)
jnz 1f /* kernel threads are uncommon */
2:
UNWIND_HINT_REGS
movq %rsp, %rdi
call syscall_return_slowpath /* returns with IRQs disabled */
TRACE_IRQS_ON /* user mode is traced as IRQS on */
@ -440,13 +451,102 @@ END(ret_from_fork)
ENTRY(irq_entries_start)
vector=FIRST_EXTERNAL_VECTOR
.rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
UNWIND_HINT_IRET_REGS
pushq $(~vector+0x80) /* Note: always in signed byte range */
vector=vector+1
jmp common_interrupt
.align 8
vector=vector+1
.endr
END(irq_entries_start)
.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
#ifdef CONFIG_DEBUG_ENTRY
pushfq
testl $X86_EFLAGS_IF, (%rsp)
jz .Lokay_\@
ud2
.Lokay_\@:
addq $8, %rsp
#endif
.endm
/*
* Enters the IRQ stack if we're not already using it. NMI-safe. Clobbers
* flags and puts old RSP into old_rsp, and leaves all other GPRs alone.
* Requires kernel GSBASE.
*
* The invariant is that, if irq_count != -1, then the IRQ stack is in use.
*/
.macro ENTER_IRQ_STACK regs=1 old_rsp
DEBUG_ENTRY_ASSERT_IRQS_OFF
movq %rsp, \old_rsp
.if \regs
UNWIND_HINT_REGS base=\old_rsp
.endif
incl PER_CPU_VAR(irq_count)
jnz .Lirq_stack_push_old_rsp_\@
/*
* Right now, if we just incremented irq_count to zero, we've
* claimed the IRQ stack but we haven't switched to it yet.
*
* If anything is added that can interrupt us here without using IST,
* it must be *extremely* careful to limit its stack usage. This
* could include kprobes and a hypothetical future IST-less #DB
* handler.
*
* The OOPS unwinder relies on the word at the top of the IRQ
* stack linking back to the previous RSP for the entire time we're
* on the IRQ stack. For this to work reliably, we need to write
* it before we actually move ourselves to the IRQ stack.
*/
movq \old_rsp, PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)
movq PER_CPU_VAR(irq_stack_ptr), %rsp
#ifdef CONFIG_DEBUG_ENTRY
/*
* If the first movq above becomes wrong due to IRQ stack layout
* changes, the only way we'll notice is if we try to unwind right
* here. Assert that we set up the stack right to catch this type
* of bug quickly.
*/
cmpq -8(%rsp), \old_rsp
je .Lirq_stack_okay\@
ud2
.Lirq_stack_okay\@:
#endif
.Lirq_stack_push_old_rsp_\@:
pushq \old_rsp
.if \regs
UNWIND_HINT_REGS indirect=1
.endif
.endm
/*
* Undoes ENTER_IRQ_STACK.
*/
.macro LEAVE_IRQ_STACK regs=1
DEBUG_ENTRY_ASSERT_IRQS_OFF
/* We need to be off the IRQ stack before decrementing irq_count. */
popq %rsp
.if \regs
UNWIND_HINT_REGS
.endif
/*
* As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming
* the irq stack but we're not on it.
*/
decl PER_CPU_VAR(irq_count)
.endm
/*
* Interrupt entry/exit.
*
@ -485,17 +585,7 @@ END(irq_entries_start)
CALL_enter_from_user_mode
1:
/*
* Save previous stack pointer, optionally switch to interrupt stack.
* irq_count is used to check if a CPU is already on an interrupt stack
* or not. While this is essentially redundant with preempt_count it is
* a little cheaper to use a separate counter in the PDA (short of
* moving irq_enter into assembly, which would be too much work)
*/
movq %rsp, %rdi
incl PER_CPU_VAR(irq_count)
cmovzq PER_CPU_VAR(irq_stack_ptr), %rsp
pushq %rdi
ENTER_IRQ_STACK old_rsp=%rdi
/* We entered an interrupt context - irqs are off: */
TRACE_IRQS_OFF
@ -515,10 +605,8 @@ common_interrupt:
ret_from_intr:
DISABLE_INTERRUPTS(CLBR_ANY)
TRACE_IRQS_OFF
decl PER_CPU_VAR(irq_count)
/* Restore saved previous stack */
popq %rsp
LEAVE_IRQ_STACK
testb $3, CS(%rsp)
jz retint_kernel
@ -561,6 +649,7 @@ restore_c_regs_and_iret:
INTERRUPT_RETURN
ENTRY(native_iret)
UNWIND_HINT_IRET_REGS
/*
* Are we returning to a stack segment from the LDT? Note: in
* 64-bit mode SS:RSP on the exception stack is always valid.
@ -633,6 +722,7 @@ native_irq_return_ldt:
orq PER_CPU_VAR(espfix_stack), %rax
SWAPGS
movq %rax, %rsp
UNWIND_HINT_IRET_REGS offset=8
/*
* At this point, we cannot write to the stack any more, but we can
@ -654,6 +744,7 @@ END(common_interrupt)
*/
.macro apicinterrupt3 num sym do_sym
ENTRY(\sym)
UNWIND_HINT_IRET_REGS
ASM_CLAC
pushq $~(\num)
.Lcommon_\sym:
@ -740,6 +831,8 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt
.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
ENTRY(\sym)
UNWIND_HINT_IRET_REGS offset=8
/* Sanity check */
.if \shift_ist != -1 && \paranoid == 0
.error "using shift_ist requires paranoid=1"
@ -763,6 +856,7 @@ ENTRY(\sym)
.else
call error_entry
.endif
UNWIND_HINT_REGS
/* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
.if \paranoid
@ -860,6 +954,7 @@ idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0
* edi: new selector
*/
ENTRY(native_load_gs_index)
FRAME_BEGIN
pushfq
DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
SWAPGS
@ -868,8 +963,9 @@ ENTRY(native_load_gs_index)
2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
SWAPGS
popfq
FRAME_END
ret
END(native_load_gs_index)
ENDPROC(native_load_gs_index)
EXPORT_SYMBOL(native_load_gs_index)
_ASM_EXTABLE(.Lgs_change, bad_gs)
@ -892,14 +988,12 @@ bad_gs:
ENTRY(do_softirq_own_stack)
pushq %rbp
mov %rsp, %rbp
incl PER_CPU_VAR(irq_count)
cmove PER_CPU_VAR(irq_stack_ptr), %rsp
push %rbp /* frame pointer backlink */
ENTER_IRQ_STACK regs=0 old_rsp=%r11
call __do_softirq
LEAVE_IRQ_STACK regs=0
leaveq
decl PER_CPU_VAR(irq_count)
ret
END(do_softirq_own_stack)
ENDPROC(do_softirq_own_stack)
#ifdef CONFIG_XEN
idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
@ -923,14 +1017,14 @@ ENTRY(xen_do_hypervisor_callback) /* do_hypervisor_callback(struct *pt_regs) */
* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
* see the correct pointer to the pt_regs
*/
UNWIND_HINT_FUNC
movq %rdi, %rsp /* we don't return, adjust the stack frame */
11: incl PER_CPU_VAR(irq_count)
movq %rsp, %rbp
cmovzq PER_CPU_VAR(irq_stack_ptr), %rsp
pushq %rbp /* frame pointer backlink */
UNWIND_HINT_REGS
ENTER_IRQ_STACK old_rsp=%r10
call xen_evtchn_do_upcall
popq %rsp
decl PER_CPU_VAR(irq_count)
LEAVE_IRQ_STACK
#ifndef CONFIG_PREEMPT
call xen_maybe_preempt_hcall
#endif
@ -951,6 +1045,7 @@ END(xen_do_hypervisor_callback)
* with its current contents: any discrepancy means we in category 1.
*/
ENTRY(xen_failsafe_callback)
UNWIND_HINT_EMPTY
movl %ds, %ecx
cmpw %cx, 0x10(%rsp)
jne 1f
@ -970,11 +1065,13 @@ ENTRY(xen_failsafe_callback)
pushq $0 /* RIP */
pushq %r11
pushq %rcx
UNWIND_HINT_IRET_REGS offset=8
jmp general_protection
1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
movq (%rsp), %rcx
movq 8(%rsp), %r11
addq $0x30, %rsp
UNWIND_HINT_IRET_REGS
pushq $-1 /* orig_ax = -1 => not a system call */
ALLOC_PT_GPREGS_ON_STACK
SAVE_C_REGS
@ -1020,6 +1117,7 @@ idtentry machine_check has_error_code=0 paranoid=1 do_sym=*machine_check_vec
* Return: ebx=0: need swapgs on exit, ebx=1: otherwise
*/
ENTRY(paranoid_entry)
UNWIND_HINT_FUNC
cld
SAVE_C_REGS 8
SAVE_EXTRA_REGS 8
@ -1047,6 +1145,7 @@ END(paranoid_entry)
* On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
*/
ENTRY(paranoid_exit)
UNWIND_HINT_REGS
DISABLE_INTERRUPTS(CLBR_ANY)
TRACE_IRQS_OFF_DEBUG
testl %ebx, %ebx /* swapgs needed? */
@ -1068,6 +1167,7 @@ END(paranoid_exit)
* Return: EBX=0: came from user mode; EBX=1: otherwise
*/
ENTRY(error_entry)
UNWIND_HINT_FUNC
cld
SAVE_C_REGS 8
SAVE_EXTRA_REGS 8
@ -1152,6 +1252,7 @@ END(error_entry)
* 0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
*/
ENTRY(error_exit)
UNWIND_HINT_REGS
DISABLE_INTERRUPTS(CLBR_ANY)
TRACE_IRQS_OFF
testl %ebx, %ebx
@ -1161,6 +1262,7 @@ END(error_exit)
/* Runs on exception stack */
ENTRY(nmi)
UNWIND_HINT_IRET_REGS
/*
* Fix up the exception frame if we're on Xen.
* PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most
@ -1232,11 +1334,13 @@ ENTRY(nmi)
cld
movq %rsp, %rdx
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
UNWIND_HINT_IRET_REGS base=%rdx offset=8
pushq 5*8(%rdx) /* pt_regs->ss */
pushq 4*8(%rdx) /* pt_regs->rsp */
pushq 3*8(%rdx) /* pt_regs->flags */
pushq 2*8(%rdx) /* pt_regs->cs */
pushq 1*8(%rdx) /* pt_regs->rip */
UNWIND_HINT_IRET_REGS
pushq $-1 /* pt_regs->orig_ax */
pushq %rdi /* pt_regs->di */
pushq %rsi /* pt_regs->si */
@ -1253,6 +1357,7 @@ ENTRY(nmi)
pushq %r13 /* pt_regs->r13 */
pushq %r14 /* pt_regs->r14 */
pushq %r15 /* pt_regs->r15 */
UNWIND_HINT_REGS
ENCODE_FRAME_POINTER
/*
@ -1407,6 +1512,7 @@ first_nmi:
.rept 5
pushq 11*8(%rsp)
.endr
UNWIND_HINT_IRET_REGS
/* Everything up to here is safe from nested NMIs */
@ -1422,6 +1528,7 @@ first_nmi:
pushq $__KERNEL_CS /* CS */
pushq $1f /* RIP */
INTERRUPT_RETURN /* continues at repeat_nmi below */
UNWIND_HINT_IRET_REGS
1:
#endif
@ -1471,6 +1578,7 @@ end_repeat_nmi:
* exceptions might do.
*/
call paranoid_entry
UNWIND_HINT_REGS
/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
movq %rsp, %rdi
@ -1508,17 +1616,19 @@ nmi_restore:
END(nmi)
ENTRY(ignore_sysret)
UNWIND_HINT_EMPTY
mov $-ENOSYS, %eax
sysret
END(ignore_sysret)
ENTRY(rewind_stack_do_exit)
UNWIND_HINT_FUNC
/* Prevent any naive code from trying to unwind to our caller. */
xorl %ebp, %ebp
movq PER_CPU_VAR(cpu_current_top_of_stack), %rax
leaq -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%rax), %rsp
leaq -PTREGS_SIZE(%rax), %rsp
UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE
call do_exit
1: jmp 1b
END(rewind_stack_do_exit)

98
arch/x86/include/asm/io.h
View File

@ -69,6 +69,9 @@ build_mmio_write(__writeb, "b", unsigned char, "q", )
build_mmio_write(__writew, "w", unsigned short, "r", )
build_mmio_write(__writel, "l", unsigned int, "r", )
#define readb readb
#define readw readw
#define readl readl
#define readb_relaxed(a) __readb(a)
#define readw_relaxed(a) __readw(a)
#define readl_relaxed(a) __readl(a)
@ -76,6 +79,9 @@ build_mmio_write(__writel, "l", unsigned int, "r", )
#define __raw_readw __readw
#define __raw_readl __readl
#define writeb writeb
#define writew writew
#define writel writel
#define writeb_relaxed(v, a) __writeb(v, a)
#define writew_relaxed(v, a) __writew(v, a)
#define writel_relaxed(v, a) __writel(v, a)
@ -88,13 +94,15 @@ build_mmio_write(__writel, "l", unsigned int, "r", )
#ifdef CONFIG_X86_64
build_mmio_read(readq, "q", unsigned long, "=r", :"memory")
build_mmio_read(__readq, "q", unsigned long, "=r", )
build_mmio_write(writeq, "q", unsigned long, "r", :"memory")
build_mmio_write(__writeq, "q", unsigned long, "r", )
#define readq_relaxed(a) readq(a)
#define writeq_relaxed(v, a) writeq(v, a)
#define readq_relaxed(a) __readq(a)
#define writeq_relaxed(v, a) __writeq(v, a)
#define __raw_readq(a) readq(a)
#define __raw_writeq(val, addr) writeq(val, addr)
#define __raw_readq __readq
#define __raw_writeq __writeq
/* Let people know that we have them */
#define readq readq
@ -119,6 +127,7 @@ static inline phys_addr_t virt_to_phys(volatile void *address)
{
return __pa(address);
}
#define virt_to_phys virt_to_phys
/**
* phys_to_virt - map physical address to virtual
@ -137,6 +146,7 @@ static inline void *phys_to_virt(phys_addr_t address)
{
return __va(address);
}
#define phys_to_virt phys_to_virt
/*
* Change "struct page" to physical address.
@ -169,11 +179,14 @@ static inline unsigned int isa_virt_to_bus(volatile void *address)
* else, you probably want one of the following.
*/
extern void __iomem *ioremap_nocache(resource_size_t offset, unsigned long size);
#define ioremap_nocache ioremap_nocache
extern void __iomem *ioremap_uc(resource_size_t offset, unsigned long size);
#define ioremap_uc ioremap_uc
extern void __iomem *ioremap_cache(resource_size_t offset, unsigned long size);
#define ioremap_cache ioremap_cache
extern void __iomem *ioremap_prot(resource_size_t offset, unsigned long size, unsigned long prot_val);
#define ioremap_prot ioremap_prot
/**
* ioremap - map bus memory into CPU space
@ -193,8 +206,10 @@ static inline void __iomem *ioremap(resource_size_t offset, unsigned long size)
{
return ioremap_nocache(offset, size);
}
#define ioremap ioremap
extern void iounmap(volatile void __iomem *addr);
#define iounmap iounmap
extern void set_iounmap_nonlazy(void);
@ -202,53 +217,6 @@ extern void set_iounmap_nonlazy(void);
#include <asm-generic/iomap.h>
/*
* Convert a virtual cached pointer to an uncached pointer
*/
#define xlate_dev_kmem_ptr(p) p
/**
* memset_io Set a range of I/O memory to a constant value
* @addr: The beginning of the I/O-memory range to set
* @val: The value to set the memory to
* @count: The number of bytes to set
*
* Set a range of I/O memory to a given value.
*/
static inline void
memset_io(volatile void __iomem *addr, unsigned char val, size_t count)
{
memset((void __force *)addr, val, count);
}
/**
* memcpy_fromio Copy a block of data from I/O memory
* @dst: The (RAM) destination for the copy
* @src: The (I/O memory) source for the data
* @count: The number of bytes to copy
*
* Copy a block of data from I/O memory.
*/
static inline void
memcpy_fromio(void *dst, const volatile void __iomem *src, size_t count)
{
memcpy(dst, (const void __force *)src, count);
}
/**
* memcpy_toio Copy a block of data into I/O memory
* @dst: The (I/O memory) destination for the copy
* @src: The (RAM) source for the data
* @count: The number of bytes to copy
*
* Copy a block of data to I/O memory.
*/
static inline void
memcpy_toio(volatile void __iomem *dst, const void *src, size_t count)
{
memcpy((void __force *)dst, src, count);
}
/*
* ISA space is 'always mapped' on a typical x86 system, no need to
* explicitly ioremap() it. The fact that the ISA IO space is mapped
@ -341,13 +309,38 @@ BUILDIO(b, b, char)
BUILDIO(w, w, short)
BUILDIO(l, , int)
#define inb inb
#define inw inw
#define inl inl
#define inb_p inb_p
#define inw_p inw_p
#define inl_p inl_p
#define insb insb
#define insw insw
#define insl insl
#define outb outb
#define outw outw
#define outl outl
#define outb_p outb_p
#define outw_p outw_p
#define outl_p outl_p
#define outsb outsb
#define outsw outsw
#define outsl outsl
extern void *xlate_dev_mem_ptr(phys_addr_t phys);
extern void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr);
#define xlate_dev_mem_ptr xlate_dev_mem_ptr
#define unxlate_dev_mem_ptr unxlate_dev_mem_ptr
extern int ioremap_change_attr(unsigned long vaddr, unsigned long size,
enum page_cache_mode pcm);
extern void __iomem *ioremap_wc(resource_size_t offset, unsigned long size);
#define ioremap_wc ioremap_wc
extern void __iomem *ioremap_wt(resource_size_t offset, unsigned long size);
#define ioremap_wt ioremap_wt
extern bool is_early_ioremap_ptep(pte_t *ptep);
@ -365,6 +358,9 @@ extern bool xen_biovec_phys_mergeable(const struct bio_vec *vec1,
#define IO_SPACE_LIMIT 0xffff
#include <asm-generic/io.h>
#undef PCI_IOBASE
#ifdef CONFIG_MTRR
extern int __must_check arch_phys_wc_index(int handle);
#define arch_phys_wc_index arch_phys_wc_index

107
arch/x86/include/asm/orc_types.h Normal file
View File

@ -0,0 +1,107 @@
/*
* Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _ORC_TYPES_H
#define _ORC_TYPES_H
#include <linux/types.h>
#include <linux/compiler.h>
/*
* The ORC_REG_* registers are base registers which are used to find other
* registers on the stack.
*
* ORC_REG_PREV_SP, also known as DWARF Call Frame Address (CFA), is the
* address of the previous frame: the caller's SP before it called the current
* function.
*
* ORC_REG_UNDEFINED means the corresponding register's value didn't change in
* the current frame.
*
* The most commonly used base registers are SP and BP -- which the previous SP
* is usually based on -- and PREV_SP and UNDEFINED -- which the previous BP is
* usually based on.
*
* The rest of the base registers are needed for special cases like entry code
* and GCC realigned stacks.
*/
#define ORC_REG_UNDEFINED 0
#define ORC_REG_PREV_SP 1
#define ORC_REG_DX 2
#define ORC_REG_DI 3
#define ORC_REG_BP 4
#define ORC_REG_SP 5
#define ORC_REG_R10 6
#define ORC_REG_R13 7
#define ORC_REG_BP_INDIRECT 8
#define ORC_REG_SP_INDIRECT 9
#define ORC_REG_MAX 15
/*
* ORC_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP (the
* caller's SP right before it made the call). Used for all callable
* functions, i.e. all C code and all callable asm functions.
*
* ORC_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset points
* to a fully populated pt_regs from a syscall, interrupt, or exception.
*
* ORC_TYPE_REGS_IRET: Used in entry code to indicate that sp_reg+sp_offset
* points to the iret return frame.
*
* The UNWIND_HINT macros are used only for the unwind_hint struct. They
* aren't used in struct orc_entry due to size and complexity constraints.
* Objtool converts them to real types when it converts the hints to orc
* entries.
*/
#define ORC_TYPE_CALL 0
#define ORC_TYPE_REGS 1
#define ORC_TYPE_REGS_IRET 2
#define UNWIND_HINT_TYPE_SAVE 3
#define UNWIND_HINT_TYPE_RESTORE 4
#ifndef __ASSEMBLY__
/*
* This struct is more or less a vastly simplified version of the DWARF Call
* Frame Information standard. It contains only the necessary parts of DWARF
* CFI, simplified for ease of access by the in-kernel unwinder. It tells the
* unwinder how to find the previous SP and BP (and sometimes entry regs) on
* the stack for a given code address. Each instance of the struct corresponds
* to one or more code locations.
*/
struct orc_entry {
s16 sp_offset;
s16 bp_offset;
unsigned sp_reg:4;
unsigned bp_reg:4;
unsigned type:2;
};
/*
* This struct is used by asm and inline asm code to manually annotate the
* location of registers on the stack for the ORC unwinder.
*
* Type can be either ORC_TYPE_* or UNWIND_HINT_TYPE_*.
*/
struct unwind_hint {
u32 ip;
s16 sp_offset;
u8 sp_reg;
u8 type;
};
#endif /* __ASSEMBLY__ */
#endif /* _ORC_TYPES_H */

3
arch/x86/include/asm/processor.h
View File

@ -22,6 +22,7 @@ struct vm86;
#include <asm/nops.h>
#include <asm/special_insns.h>
#include <asm/fpu/types.h>
#include <asm/unwind_hints.h>
#include <linux/personality.h>
#include <linux/cache.h>
@ -684,6 +685,7 @@ static inline void sync_core(void)
unsigned int tmp;
asm volatile (
UNWIND_HINT_SAVE
"mov %%ss, %0\n\t"
"pushq %q0\n\t"
"pushq %%rsp\n\t"
@ -693,6 +695,7 @@ static inline void sync_core(void)
"pushq %q0\n\t"
"pushq $1f\n\t"
"iretq\n\t"
UNWIND_HINT_RESTORE
"1:"
: "=&r" (tmp), "+r" (__sp) : : "cc", "memory");
#endif

41
arch/x86/include/asm/rmwcc.h
View File

@ -1,45 +1,56 @@
#ifndef _ASM_X86_RMWcc
#define _ASM_X86_RMWcc
#define __CLOBBERS_MEM "memory"
#define __CLOBBERS_MEM_CC_CX "memory", "cc", "cx"
#if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CC_HAVE_ASM_GOTO)
/* Use asm goto */
#define __GEN_RMWcc(fullop, var, cc, ...) \
#define __GEN_RMWcc(fullop, var, cc, clobbers, ...) \
do { \
asm_volatile_goto (fullop "; j" #cc " %l[cc_label]" \
: : "m" (var), ## __VA_ARGS__ \
: "memory" : cc_label); \
: : [counter] "m" (var), ## __VA_ARGS__ \
: clobbers : cc_label); \
return 0; \
cc_label: \
return 1; \
} while (0)
#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
__GEN_RMWcc(op " " arg0, var, cc)
#define __BINARY_RMWcc_ARG " %1, "
#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
__GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val))
#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */
/* Use flags output or a set instruction */
#define __GEN_RMWcc(fullop, var, cc, ...) \
#define __GEN_RMWcc(fullop, var, cc, clobbers, ...) \
do { \
bool c; \
asm volatile (fullop ";" CC_SET(cc) \
: "+m" (var), CC_OUT(cc) (c) \
: __VA_ARGS__ : "memory"); \
: [counter] "+m" (var), CC_OUT(cc) (c) \
: __VA_ARGS__ : clobbers); \
return c; \
} while (0)
#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
__GEN_RMWcc(op " " arg0, var, cc)
#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
__GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val))
#define __BINARY_RMWcc_ARG " %2, "
#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */
#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
__GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM)
#define GEN_UNARY_SUFFIXED_RMWcc(op, suffix, var, arg0, cc) \
__GEN_RMWcc(op " " arg0 "\n\t" suffix, var, cc, \
__CLOBBERS_MEM_CC_CX)
#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
__GEN_RMWcc(op __BINARY_RMWcc_ARG arg0, var, cc, \
__CLOBBERS_MEM, vcon (val))
#define GEN_BINARY_SUFFIXED_RMWcc(op, suffix, var, vcon, val, arg0, cc) \
__GEN_RMWcc(op __BINARY_RMWcc_ARG arg0 "\n\t" suffix, var, cc, \
__CLOBBERS_MEM_CC_CX, vcon (val))
#endif /* _ASM_X86_RMWcc */

103
arch/x86/include/asm/unwind_hints.h Normal file
View File

@ -0,0 +1,103 @@
#ifndef _ASM_X86_UNWIND_HINTS_H
#define _ASM_X86_UNWIND_HINTS_H
#include "orc_types.h"
#ifdef __ASSEMBLY__
/*
* In asm, there are two kinds of code: normal C-type callable functions and
* the rest. The normal callable functions can be called by other code, and
* don't do anything unusual with the stack. Such normal callable functions
* are annotated with the ENTRY/ENDPROC macros. Most asm code falls in this
* category. In this case, no special debugging annotations are needed because
* objtool can automatically generate the ORC data for the ORC unwinder to read
* at runtime.
*
* Anything which doesn't fall into the above category, such as syscall and
* interrupt handlers, tends to not be called directly by other functions, and
* often does unusual non-C-function-type things with the stack pointer. Such
* code needs to be annotated such that objtool can understand it. The
* following CFI hint macros are for this type of code.
*
* These macros provide hints to objtool about the state of the stack at each
* instruction. Objtool starts from the hints and follows the code flow,
* making automatic CFI adjustments when it sees pushes and pops, filling out
* the debuginfo as necessary. It will also warn if it sees any
* inconsistencies.
*/
.macro UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=0 type=ORC_TYPE_CALL
#ifdef CONFIG_STACK_VALIDATION
.Lunwind_hint_ip_\@:
.pushsection .discard.unwind_hints
/* struct unwind_hint */
.long .Lunwind_hint_ip_\@ - .
.short \sp_offset
.byte \sp_reg
.byte \type
.popsection
#endif
.endm
.macro UNWIND_HINT_EMPTY
UNWIND_HINT sp_reg=ORC_REG_UNDEFINED
.endm
.macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 iret=0
.if \base == %rsp && \indirect
.set sp_reg, ORC_REG_SP_INDIRECT
.elseif \base == %rsp
.set sp_reg, ORC_REG_SP
.elseif \base == %rbp
.set sp_reg, ORC_REG_BP
.elseif \base == %rdi
.set sp_reg, ORC_REG_DI
.elseif \base == %rdx
.set sp_reg, ORC_REG_DX
.elseif \base == %r10
.set sp_reg, ORC_REG_R10
.else
.error "UNWIND_HINT_REGS: bad base register"
.endif
.set sp_offset, \offset
.if \iret
.set type, ORC_TYPE_REGS_IRET
.elseif \extra == 0
.set type, ORC_TYPE_REGS_IRET
.set sp_offset, \offset + (16*8)
.else
.set type, ORC_TYPE_REGS
.endif
UNWIND_HINT sp_reg=sp_reg sp_offset=sp_offset type=type
.endm
.macro UNWIND_HINT_IRET_REGS base=%rsp offset=0
UNWIND_HINT_REGS base=\base offset=\offset iret=1
.endm
.macro UNWIND_HINT_FUNC sp_offset=8
UNWIND_HINT sp_offset=\sp_offset
.endm
#else /* !__ASSEMBLY__ */
#define UNWIND_HINT(sp_reg, sp_offset, type) \
"987: \n\t" \
".pushsection .discard.unwind_hints\n\t" \
/* struct unwind_hint */ \
".long 987b - .\n\t" \
".short " __stringify(sp_offset) "\n\t" \
".byte " __stringify(sp_reg) "\n\t" \
".byte " __stringify(type) "\n\t" \
".popsection\n\t"
#define UNWIND_HINT_SAVE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_SAVE)
#define UNWIND_HINT_RESTORE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_RESTORE)
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_UNWIND_HINTS_H */

12
arch/x86/kernel/dumpstack.c
View File

@ -94,6 +94,9 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
if (stack_name)
printk("%s <%s>\n", log_lvl, stack_name);
if (regs && on_stack(&stack_info, regs, sizeof(*regs)))
__show_regs(regs, 0);
/*
* Scan the stack, printing any text addresses we find. At the
* same time, follow proper stack frames with the unwinder.
@ -118,10 +121,8 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
* Don't print regs->ip again if it was already printed
* by __show_regs() below.
*/
if (regs && stack == &regs->ip) {
unwind_next_frame(&state);
continue;
}
if (regs && stack == &regs->ip)
goto next;
if (stack == ret_addr_p)
reliable = 1;
@ -144,6 +145,7 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
if (!reliable)
continue;
next:
/*
* Get the next frame from the unwinder. No need to
* check for an error: if anything goes wrong, the rest
@ -153,7 +155,7 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
/* if the frame has entry regs, print them */
regs = unwind_get_entry_regs(&state);
if (regs)
if (regs && on_stack(&stack_info, regs, sizeof(*regs)))
__show_regs(regs, 0);
}

4
arch/x86/kernel/dumpstack_32.c
View File

@ -37,7 +37,7 @@ static bool in_hardirq_stack(unsigned long *stack, struct stack_info *info)
* This is a software stack, so 'end' can be a valid stack pointer.
* It just means the stack is empty.
*/
if (stack < begin || stack > end)
if (stack <= begin || stack > end)
return false;
info->type = STACK_TYPE_IRQ;
@ -62,7 +62,7 @@ static bool in_softirq_stack(unsigned long *stack, struct stack_info *info)
* This is a software stack, so 'end' can be a valid stack pointer.
* It just means the stack is empty.
*/
if (stack < begin || stack > end)
if (stack <= begin || stack > end)
return false;
info->type = STACK_TYPE_SOFTIRQ;

4
arch/x86/kernel/dumpstack_64.c
View File

@ -55,7 +55,7 @@ static bool in_exception_stack(unsigned long *stack, struct stack_info *info)
begin = end - (exception_stack_sizes[k] / sizeof(long));
regs = (struct pt_regs *)end - 1;
if (stack < begin || stack >= end)
if (stack <= begin || stack >= end)
continue;
info->type = STACK_TYPE_EXCEPTION + k;
@ -78,7 +78,7 @@ static bool in_irq_stack(unsigned long *stack, struct stack_info *info)
* This is a software stack, so 'end' can be a valid stack pointer.
* It just means the stack is empty.
*/
if (stack < begin || stack > end)
if (stack <= begin || stack > end)
return false;
info->type = STACK_TYPE_IRQ;

3
arch/x86/kernel/process_64.c
View File

@ -279,6 +279,9 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
unsigned prev_fsindex, prev_gsindex;
WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
this_cpu_read(irq_count) != -1);
switch_fpu_prepare(prev_fpu, cpu);
/* We must save %fs and %gs before load_TLS() because

27
include/asm-generic/io.h
View File

@ -915,6 +915,9 @@ extern void ioport_unmap(void __iomem *p);
#endif /* CONFIG_GENERIC_IOMAP */
#endif /* CONFIG_HAS_IOPORT_MAP */
/*
* Convert a virtual cached pointer to an uncached pointer
*/
#ifndef xlate_dev_kmem_ptr
#define xlate_dev_kmem_ptr xlate_dev_kmem_ptr
static inline void *xlate_dev_kmem_ptr(void *addr)
@ -954,6 +957,14 @@ static inline void *bus_to_virt(unsigned long address)
#ifndef memset_io
#define memset_io memset_io
/**
* memset_io Set a range of I/O memory to a constant value
* @addr: The beginning of the I/O-memory range to set
* @val: The value to set the memory to
* @count: The number of bytes to set
*
* Set a range of I/O memory to a given value.
*/
static inline void memset_io(volatile void __iomem *addr, int value,
size_t size)
{
@ -963,6 +974,14 @@ static inline void memset_io(volatile void __iomem *addr, int value,
#ifndef memcpy_fromio
#define memcpy_fromio memcpy_fromio
/**
* memcpy_fromio Copy a block of data from I/O memory
* @dst: The (RAM) destination for the copy
* @src: The (I/O memory) source for the data
* @count: The number of bytes to copy
*
* Copy a block of data from I/O memory.
*/
static inline void memcpy_fromio(void *buffer,
const volatile void __iomem *addr,
size_t size)
@ -973,6 +992,14 @@ static inline void memcpy_fromio(void *buffer,
#ifndef memcpy_toio
#define memcpy_toio memcpy_toio
/**
* memcpy_toio Copy a block of data into I/O memory
* @dst: The (I/O memory) destination for the copy
* @src: The (RAM) source for the data
* @count: The number of bytes to copy
*
* Copy a block of data to I/O memory.
*/
static inline void memcpy_toio(volatile void __iomem *addr, const void *buffer,
size_t size)
{

13
include/linux/compiler-gcc.h
View File

@ -203,11 +203,16 @@
#ifdef CONFIG_STACK_VALIDATION
#define annotate_unreachable() ({ \
asm("%c0:\t\n" \
".pushsection .discard.unreachable\t\n" \
".long %c0b - .\t\n" \
".popsection\t\n" : : "i" (__LINE__)); \
asm("%c0:\n\t" \
".pushsection .discard.unreachable\n\t" \
".long %c0b - .\n\t" \
".popsection\n\t" : : "i" (__LINE__)); \
})
#define ASM_UNREACHABLE \
"999:\n\t" \
".pushsection .discard.unreachable\n\t" \
".long 999b - .\n\t" \
".popsection\n\t"
#else
#define annotate_unreachable()
#endif

3
include/linux/compiler.h
View File

@ -185,6 +185,9 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
#endif
/* Unreachable code */
#ifndef ASM_UNREACHABLE
# define ASM_UNREACHABLE
#endif
#ifndef unreachable
# define unreachable() do { } while (1)
#endif

3
scripts/Makefile.build
View File

@ -262,6 +262,9 @@ objtool_args = check
ifndef CONFIG_FRAME_POINTER
objtool_args += --no-fp
endif
ifdef CONFIG_GCOV_KERNEL
objtool_args += --no-unreachable
endif
# 'OBJECT_FILES_NON_STANDARD := y': skip objtool checking for a directory
# 'OBJECT_FILES_NON_STANDARD_foo.o := 'y': skip objtool checking for a file

3
tools/objtool/Build
View File

@ -1,6 +1,9 @@
objtool-y += arch/$(SRCARCH)/
objtool-y += builtin-check.o
objtool-y += builtin-orc.o
objtool-y += check.o
objtool-y += orc_gen.o
objtool-y += orc_dump.o
objtool-y += elf.o
objtool-y += special.o
objtool-y += objtool.o

56
tools/objtool/Documentation/stack-validation.txt
View File

@ -11,9 +11,6 @@ analyzes every .o file and ensures the validity of its stack metadata.
It enforces a set of rules on asm code and C inline assembly code so
that stack traces can be reliable.
Currently it only checks frame pointer usage, but there are plans to add
CFI validation for C files and CFI generation for asm files.
For each function, it recursively follows all possible code paths and
validates the correct frame pointer state at each instruction.
@ -23,6 +20,10 @@ alternative execution paths to a given instruction (or set of
instructions). Similarly, it knows how to follow switch statements, for
which gcc sometimes uses jump tables.
(Objtool also has an 'orc generate' subcommand which generates debuginfo
for the ORC unwinder. See Documentation/x86/orc-unwinder.txt in the
kernel tree for more details.)
Why do we need stack metadata validation?
-----------------------------------------
@ -93,37 +94,14 @@ a) More reliable stack traces for frame pointer enabled kernels
or at the very end of the function after the stack frame has been
destroyed. This is an inherent limitation of frame pointers.
b) 100% reliable stack traces for DWARF enabled kernels
b) ORC (Oops Rewind Capability) unwind table generation
(NOTE: This is not yet implemented)
An alternative to frame pointers and DWARF, ORC unwind data can be
used to walk the stack. Unlike frame pointers, ORC data is out of
band. So it doesn't affect runtime performance and it can be
reliable even when interrupts or exceptions are involved.
As an alternative to frame pointers, DWARF Call Frame Information
(CFI) metadata can be used to walk the stack. Unlike frame pointers,
CFI metadata is out of band. So it doesn't affect runtime
performance and it can be reliable even when interrupts or exceptions
are involved.
For C code, gcc automatically generates DWARF CFI metadata. But for
asm code, generating CFI is a tedious manual approach which requires
manually placed .cfi assembler macros to be scattered throughout the
code. It's clumsy and very easy to get wrong, and it makes the real
code harder to read.
Stacktool will improve this situation in several ways. For code
which already has CFI annotations, it will validate them. For code
which doesn't have CFI annotations, it will generate them. So an
architecture can opt to strip out all the manual .cfi annotations
from their asm code and have objtool generate them instead.
We might also add a runtime stack validation debug option where we
periodically walk the stack from schedule() and/or an NMI to ensure
that the stack metadata is sane and that we reach the bottom of the
stack.
So the benefit of objtool here will be that external tooling should
always show perfect stack traces. And the same will be true for
kernel warning/oops traces if the architecture has a runtime DWARF
unwinder.
For more details, see Documentation/x86/orc-unwinder.txt.
c) Higher live patching compatibility rate
@ -211,7 +189,7 @@ they mean, and suggestions for how to fix them.
function, add proper frame pointer logic using the FRAME_BEGIN and
FRAME_END macros. Otherwise, if it's not a callable function, remove
its ELF function annotation by changing ENDPROC to END, and instead
use the manual CFI hint macros in asm/undwarf.h.
use the manual unwind hint macros in asm/unwind_hints.h.
If it's a GCC-compiled .c file, the error may be because the function
uses an inline asm() statement which has a "call" instruction. An
@ -231,8 +209,8 @@ they mean, and suggestions for how to fix them.
If the error is for an asm file, and the instruction is inside (or
reachable from) a callable function, the function should be annotated
with the ENTRY/ENDPROC macros (ENDPROC is the important one).
Otherwise, the code should probably be annotated with the CFI hint
macros in asm/undwarf.h so objtool and the unwinder can know the
Otherwise, the code should probably be annotated with the unwind hint
macros in asm/unwind_hints.h so objtool and the unwinder can know the
stack state associated with the code.
If you're 100% sure the code won't affect stack traces, or if you're
@ -258,7 +236,7 @@ they mean, and suggestions for how to fix them.
instructions aren't allowed in a callable function, and are most
likely part of the kernel entry code. They should usually not have
the callable function annotation (ENDPROC) and should always be
annotated with the CFI hint macros in asm/undwarf.h.
annotated with the unwind hint macros in asm/unwind_hints.h.
6. file.o: warning: objtool: func()+0x26: sibling call from callable instruction with modified stack frame
@ -272,7 +250,7 @@ they mean, and suggestions for how to fix them.
If the instruction is not actually in a callable function (e.g.
kernel entry code), change ENDPROC to END and annotate manually with
the CFI hint macros in asm/undwarf.h.
the unwind hint macros in asm/unwind_hints.h.
7. file: warning: objtool: func()+0x5c: stack state mismatch
@ -288,8 +266,8 @@ they mean, and suggestions for how to fix them.
Another possibility is that the code has some asm or inline asm which
does some unusual things to the stack or the frame pointer. In such
cases it's probably appropriate to use the CFI hint macros in
asm/undwarf.h.
cases it's probably appropriate to use the unwind hint macros in
asm/unwind_hints.h.
8. file.o: warning: objtool: funcA() falls through to next function funcB()

3
tools/objtool/Makefile
View File

@ -52,6 +52,9 @@ $(OBJTOOL): $(LIBSUBCMD) $(OBJTOOL_IN)
diff -I'^#include' arch/x86/insn/inat.h ../../arch/x86/include/asm/inat.h >/dev/null && \
diff -I'^#include' arch/x86/insn/inat_types.h ../../arch/x86/include/asm/inat_types.h >/dev/null) \
|| echo "warning: objtool: x86 instruction decoder differs from kernel" >&2 )) || true
@(test -d ../../kernel -a -d ../../tools -a -d ../objtool && (( \
diff ../../arch/x86/include/asm/orc_types.h orc_types.h >/dev/null) \
|| echo "warning: objtool: orc_types.h differs from kernel" >&2 )) || true
$(QUIET_LINK)$(CC) $(OBJTOOL_IN) $(LDFLAGS) -o $@

7
tools/objtool/builtin-check.c
View File

@ -29,7 +29,7 @@
#include "builtin.h"
#include "check.h"
bool nofp;
bool no_fp, no_unreachable;
static const char * const check_usage[] = {
"objtool check [<options>] file.o",
@ -37,7 +37,8 @@ static const char * const check_usage[] = {
};
const struct option check_options[] = {
OPT_BOOLEAN('f', "no-fp", &nofp, "Skip frame pointer validation"),
OPT_BOOLEAN('f', "no-fp", &no_fp, "Skip frame pointer validation"),
OPT_BOOLEAN('u', "no-unreachable", &no_unreachable, "Skip 'unreachable instruction' warnings"),
OPT_END(),
};
@ -52,5 +53,5 @@ int cmd_check(int argc, const char **argv)
objname = argv[0];
return check(objname, nofp);
return check(objname, no_fp, no_unreachable, false);
}

70
tools/objtool/builtin-orc.c Normal file
View File

@ -0,0 +1,70 @@
/*
* Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
/*
* objtool orc:
*
* This command analyzes a .o file and adds .orc_unwind and .orc_unwind_ip
* sections to it, which is used by the in-kernel ORC unwinder.
*
* This command is a superset of "objtool check".
*/
#include <string.h>
#include <subcmd/parse-options.h>
#include "builtin.h"
#include "check.h"
static const char *orc_usage[] = {
"objtool orc generate [<options>] file.o",
"objtool orc dump file.o",
NULL,
};
extern const struct option check_options[];
extern bool no_fp, no_unreachable;
int cmd_orc(int argc, const char **argv)
{
const char *objname;
argc--; argv++;
if (!strncmp(argv[0], "gen", 3)) {
argc = parse_options(argc, argv, check_options, orc_usage, 0);
if (argc != 1)
usage_with_options(orc_usage, check_options);
objname = argv[0];
return check(objname, no_fp, no_unreachable, true);
}
if (!strcmp(argv[0], "dump")) {
if (argc != 2)
usage_with_options(orc_usage, check_options);
objname = argv[1];
return orc_dump(objname);
}
usage_with_options(orc_usage, check_options);
return 0;
}

1
tools/objtool/builtin.h
View File

@ -18,5 +18,6 @@
#define _BUILTIN_H
extern int cmd_check(int argc, const char **argv);
extern int cmd_orc(int argc, const char **argv);
#endif /* _BUILTIN_H */

281
tools/objtool/check.c
View File

@ -33,11 +33,11 @@ struct alternative {
};
const char *objname;
static bool nofp;
static bool no_fp;
struct cfi_state initial_func_cfi;
static struct instruction *find_insn(struct objtool_file *file,
struct section *sec, unsigned long offset)
struct instruction *find_insn(struct objtool_file *file,
struct section *sec, unsigned long offset)
{
struct instruction *insn;
@ -59,19 +59,6 @@ static struct instruction *next_insn_same_sec(struct objtool_file *file,
return next;
}
static bool gcov_enabled(struct objtool_file *file)
{
struct section *sec;
struct symbol *sym;
for_each_sec(file, sec)
list_for_each_entry(sym, &sec->symbol_list, list)
if (!strncmp(sym->name, "__gcov_.", 8))
return true;
return false;
}
#define func_for_each_insn(file, func, insn) \
for (insn = find_insn(file, func->sec, func->offset); \
insn && &insn->list != &file->insn_list && \
@ -100,7 +87,6 @@ static bool gcov_enabled(struct objtool_file *file)
static bool ignore_func(struct objtool_file *file, struct symbol *func)
{
struct rela *rela;
struct instruction *insn;
/* check for STACK_FRAME_NON_STANDARD */
if (file->whitelist && file->whitelist->rela)
@ -113,11 +99,6 @@ static bool ignore_func(struct objtool_file *file, struct symbol *func)
return true;
}
/* check if it has a context switching instruction */
func_for_each_insn(file, func, insn)
if (insn->type == INSN_CONTEXT_SWITCH)
return true;
return false;
}
@ -259,6 +240,11 @@ static int decode_instructions(struct objtool_file *file)
if (!(sec->sh.sh_flags & SHF_EXECINSTR))
continue;
if (strcmp(sec->name, ".altinstr_replacement") &&
strcmp(sec->name, ".altinstr_aux") &&
strncmp(sec->name, ".discard.", 9))
sec->text = true;
for (offset = 0; offset < sec->len; offset += insn->len) {
insn = malloc(sizeof(*insn));
if (!insn) {
@ -874,6 +860,99 @@ static int add_switch_table_alts(struct objtool_file *file)
return 0;
}
static int read_unwind_hints(struct objtool_file *file)
{
struct section *sec, *relasec;
struct rela *rela;
struct unwind_hint *hint;
struct instruction *insn;
struct cfi_reg *cfa;
int i;
sec = find_section_by_name(file->elf, ".discard.unwind_hints");
if (!sec)
return 0;
relasec = sec->rela;
if (!relasec) {
WARN("missing .rela.discard.unwind_hints section");
return -1;
}
if (sec->len % sizeof(struct unwind_hint)) {
WARN("struct unwind_hint size mismatch");
return -1;
}
file->hints = true;
for (i = 0; i < sec->len / sizeof(struct unwind_hint); i++) {
hint = (struct unwind_hint *)sec->data->d_buf + i;
rela = find_rela_by_dest(sec, i * sizeof(*hint));
if (!rela) {
WARN("can't find rela for unwind_hints[%d]", i);
return -1;
}
insn = find_insn(file, rela->sym->sec, rela->addend);
if (!insn) {
WARN("can't find insn for unwind_hints[%d]", i);
return -1;
}
cfa = &insn->state.cfa;
if (hint->type == UNWIND_HINT_TYPE_SAVE) {
insn->save = true;
continue;
} else if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
insn->restore = true;
insn->hint = true;
continue;
}
insn->hint = true;
switch (hint->sp_reg) {
case ORC_REG_UNDEFINED:
cfa->base = CFI_UNDEFINED;
break;
case ORC_REG_SP:
cfa->base = CFI_SP;
break;
case ORC_REG_BP:
cfa->base = CFI_BP;
break;
case ORC_REG_SP_INDIRECT:
cfa->base = CFI_SP_INDIRECT;
break;
case ORC_REG_R10:
cfa->base = CFI_R10;
break;
case ORC_REG_R13:
cfa->base = CFI_R13;
break;
case ORC_REG_DI:
cfa->base = CFI_DI;
break;
case ORC_REG_DX:
cfa->base = CFI_DX;
break;
default:
WARN_FUNC("unsupported unwind_hint sp base reg %d",
insn->sec, insn->offset, hint->sp_reg);
return -1;
}
cfa->offset = hint->sp_offset;
insn->state.type = hint->type;
}
return 0;
}
static int decode_sections(struct objtool_file *file)
{
int ret;
@ -904,6 +983,10 @@ static int decode_sections(struct objtool_file *file)
if (ret)
return ret;
ret = read_unwind_hints(file);
if (ret)
return ret;
return 0;
}
@ -947,6 +1030,30 @@ static bool has_valid_stack_frame(struct insn_state *state)
return false;
}
static int update_insn_state_regs(struct instruction *insn, struct insn_state *state)
{
struct cfi_reg *cfa = &state->cfa;
struct stack_op *op = &insn->stack_op;
if (cfa->base != CFI_SP)
return 0;
/* push */
if (op->dest.type == OP_DEST_PUSH)
cfa->offset += 8;
/* pop */
if (op->src.type == OP_SRC_POP)
cfa->offset -= 8;
/* add immediate to sp */
if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
cfa->offset -= op->src.offset;
return 0;
}
static void save_reg(struct insn_state *state, unsigned char reg, int base,
int offset)
{
@ -1032,6 +1139,9 @@ static int update_insn_state(struct instruction *insn, struct insn_state *state)
return 0;
}
if (state->type == ORC_TYPE_REGS || state->type == ORC_TYPE_REGS_IRET)
return update_insn_state_regs(insn, state);
switch (op->dest.type) {
case OP_DEST_REG:
@ -1051,7 +1161,7 @@ static int update_insn_state(struct instruction *insn, struct insn_state *state)
regs[CFI_BP].base = CFI_BP;
regs[CFI_BP].offset = -state->stack_size;
state->bp_scratch = false;
} else if (!nofp) {
} else if (!no_fp) {
WARN_FUNC("unknown stack-related register move",
insn->sec, insn->offset);
@ -1222,7 +1332,7 @@ static int update_insn_state(struct instruction *insn, struct insn_state *state)
}
/* detect when asm code uses rbp as a scratch register */
if (!nofp && insn->func && op->src.reg == CFI_BP &&
if (!no_fp && insn->func && op->src.reg == CFI_BP &&
cfa->base != CFI_BP)
state->bp_scratch = true;
break;
@ -1323,6 +1433,10 @@ static bool insn_state_match(struct instruction *insn, struct insn_state *state)
break;
}
} else if (state1->type != state2->type) {
WARN_FUNC("stack state mismatch: type1=%d type2=%d",
insn->sec, insn->offset, state1->type, state2->type);
} else if (state1->drap != state2->drap ||
(state1->drap && state1->drap_reg != state2->drap_reg)) {
WARN_FUNC("stack state mismatch: drap1=%d(%d) drap2=%d(%d)",
@ -1346,7 +1460,7 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
struct insn_state state)
{
struct alternative *alt;
struct instruction *insn;
struct instruction *insn, *next_insn;
struct section *sec;
struct symbol *func = NULL;
int ret;
@ -1361,6 +1475,8 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
}
while (1) {
next_insn = next_insn_same_sec(file, insn);
if (file->c_file && insn->func) {
if (func && func != insn->func) {
WARN("%s() falls through to next function %s()",
@ -1378,13 +1494,54 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
}
if (insn->visited) {
if (!!insn_state_match(insn, &state))
if (!insn->hint && !insn_state_match(insn, &state))
return 1;
return 0;
}
insn->state = state;
if (insn->hint) {
if (insn->restore) {
struct instruction *save_insn, *i;
i = insn;
save_insn = NULL;
func_for_each_insn_continue_reverse(file, func, i) {
if (i->save) {
save_insn = i;
break;
}
}
if (!save_insn) {
WARN_FUNC("no corresponding CFI save for CFI restore",
sec, insn->offset);
return 1;
}
if (!save_insn->visited) {
/*
* Oops, no state to copy yet.
* Hopefully we can reach this
* instruction from another branch
* after the save insn has been
* visited.
*/
if (insn == first)
return 0;
WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo",
sec, insn->offset);
return 1;
}
insn->state = save_insn->state;
}
state = insn->state;
} else
insn->state = state;
insn->visited = true;
@ -1423,7 +1580,7 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
/* fallthrough */
case INSN_CALL_DYNAMIC:
if (!nofp && func && !has_valid_stack_frame(&state)) {
if (!no_fp && func && !has_valid_stack_frame(&state)) {
WARN_FUNC("call without frame pointer save/setup",
sec, insn->offset);
return 1;
@ -1461,6 +1618,14 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
return 0;
case INSN_CONTEXT_SWITCH:
if (func && (!next_insn || !next_insn->hint)) {
WARN_FUNC("unsupported instruction in callable function",
sec, insn->offset);
return 1;
}
return 0;
case INSN_STACK:
if (update_insn_state(insn, &state))
return -1;
@ -1474,7 +1639,7 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
if (insn->dead_end)
return 0;
insn = next_insn_same_sec(file, insn);
insn = next_insn;
if (!insn) {
WARN("%s: unexpected end of section", sec->name);
return 1;
@ -1484,6 +1649,27 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
return 0;
}
static int validate_unwind_hints(struct objtool_file *file)
{
struct instruction *insn;
int ret, warnings = 0;
struct insn_state state;
if (!file->hints)
return 0;
clear_insn_state(&state);
for_each_insn(file, insn) {
if (insn->hint && !insn->visited) {
ret = validate_branch(file, insn, state);
warnings += ret;
}
}
return warnings;
}
static bool is_kasan_insn(struct instruction *insn)
{
return (insn->type == INSN_CALL &&
@ -1580,15 +1766,6 @@ static int validate_reachable_instructions(struct objtool_file *file)
if (insn->visited || ignore_unreachable_insn(insn))
continue;
/*
* gcov produces a lot of unreachable instructions. If we get
* an unreachable warning and the file has gcov enabled, just
* ignore it, and all other such warnings for the file. Do
* this here because this is an expensive function.
*/
if (gcov_enabled(file))
return 0;
WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
return 1;
}
@ -1613,15 +1790,15 @@ static void cleanup(struct objtool_file *file)
elf_close(file->elf);
}
int check(const char *_objname, bool _nofp)
int check(const char *_objname, bool _no_fp, bool no_unreachable, bool orc)
{
struct objtool_file file;
int ret, warnings = 0;
objname = _objname;
nofp = _nofp;
no_fp = _no_fp;
file.elf = elf_open(objname);
file.elf = elf_open(objname, orc ? O_RDWR : O_RDONLY);
if (!file.elf)
return 1;
@ -1629,8 +1806,9 @@ int check(const char *_objname, bool _nofp)
hash_init(file.insn_hash);
file.whitelist = find_section_by_name(file.elf, ".discard.func_stack_frame_non_standard");
file.rodata = find_section_by_name(file.elf, ".rodata");
file.ignore_unreachables = false;
file.c_file = find_section_by_name(file.elf, ".comment");
file.ignore_unreachables = no_unreachable;
file.hints = false;
arch_initial_func_cfi_state(&initial_func_cfi);
@ -1647,6 +1825,11 @@ int check(const char *_objname, bool _nofp)
goto out;
warnings += ret;
ret = validate_unwind_hints(&file);
if (ret < 0)
goto out;
warnings += ret;
if (!warnings) {
ret = validate_reachable_instructions(&file);
if (ret < 0)
@ -1654,6 +1837,20 @@ int check(const char *_objname, bool _nofp)
warnings += ret;
}
if (orc) {
ret = create_orc(&file);
if (ret < 0)
goto out;
ret = create_orc_sections(&file);
if (ret < 0)
goto out;
ret = elf_write(file.elf);
if (ret < 0)
goto out;
}
out:
cleanup(&file);

19
tools/objtool/check.h
View File

@ -22,12 +22,14 @@
#include "elf.h"
#include "cfi.h"
#include "arch.h"
#include "orc.h"
#include <linux/hashtable.h>
struct insn_state {
struct cfi_reg cfa;
struct cfi_reg regs[CFI_NUM_REGS];
int stack_size;
unsigned char type;
bool bp_scratch;
bool drap;
int drap_reg;
@ -41,13 +43,14 @@ struct instruction {
unsigned int len;
unsigned char type;
unsigned long immediate;
bool alt_group, visited, dead_end, ignore;
bool alt_group, visited, dead_end, ignore, hint, save, restore;
struct symbol *call_dest;
struct instruction *jump_dest;
struct list_head alts;
struct symbol *func;
struct stack_op stack_op;
struct insn_state state;
struct orc_entry orc;
};
struct objtool_file {
@ -55,12 +58,22 @@ struct objtool_file {
struct list_head insn_list;
DECLARE_HASHTABLE(insn_hash, 16);
struct section *rodata, *whitelist;
bool ignore_unreachables, c_file;
bool ignore_unreachables, c_file, hints;
};
int check(const char *objname, bool nofp);
int check(const char *objname, bool no_fp, bool no_unreachable, bool orc);
struct instruction *find_insn(struct objtool_file *file,
struct section *sec, unsigned long offset);
#define for_each_insn(file, insn) \
list_for_each_entry(insn, &file->insn_list, list)
#define sec_for_each_insn(file, sec, insn) \
for (insn = find_insn(file, sec, 0); \
insn && &insn->list != &file->insn_list && \
insn->sec == sec; \
insn = list_next_entry(insn, list))
#endif /* _CHECK_H */

212
tools/objtool/elf.c
View File

@ -30,16 +30,6 @@
#include "elf.h"
#include "warn.h"
/*
* Fallback for systems without this "read, mmaping if possible" cmd.
*/
#ifndef ELF_C_READ_MMAP
#define ELF_C_READ_MMAP ELF_C_READ
#endif
#define WARN_ELF(format, ...) \
WARN(format ": %s", ##__VA_ARGS__, elf_errmsg(-1))
struct section *find_section_by_name(struct elf *elf, const char *name)
{
struct section *sec;
@ -349,9 +339,10 @@ static int read_relas(struct elf *elf)
return 0;
}
struct elf *elf_open(const char *name)
struct elf *elf_open(const char *name, int flags)
{
struct elf *elf;
Elf_Cmd cmd;
elf_version(EV_CURRENT);
@ -364,13 +355,20 @@ struct elf *elf_open(const char *name)
INIT_LIST_HEAD(&elf->sections);
elf->fd = open(name, O_RDONLY);
elf->fd = open(name, flags);
if (elf->fd == -1) {
perror("open");
goto err;
}
elf->elf = elf_begin(elf->fd, ELF_C_READ_MMAP, NULL);
if ((flags & O_ACCMODE) == O_RDONLY)
cmd = ELF_C_READ_MMAP;
else if ((flags & O_ACCMODE) == O_RDWR)
cmd = ELF_C_RDWR;
else /* O_WRONLY */
cmd = ELF_C_WRITE;
elf->elf = elf_begin(elf->fd, cmd, NULL);
if (!elf->elf) {
WARN_ELF("elf_begin");
goto err;
@ -397,6 +395,194 @@ err:
return NULL;
}
struct section *elf_create_section(struct elf *elf, const char *name,
size_t entsize, int nr)
{
struct section *sec, *shstrtab;
size_t size = entsize * nr;
struct Elf_Scn *s;
Elf_Data *data;
sec = malloc(sizeof(*sec));
if (!sec) {
perror("malloc");
return NULL;
}
memset(sec, 0, sizeof(*sec));
INIT_LIST_HEAD(&sec->symbol_list);
INIT_LIST_HEAD(&sec->rela_list);
hash_init(sec->rela_hash);
hash_init(sec->symbol_hash);
list_add_tail(&sec->list, &elf->sections);
s = elf_newscn(elf->elf);
if (!s) {
WARN_ELF("elf_newscn");
return NULL;
}
sec->name = strdup(name);
if (!sec->name) {
perror("strdup");
return NULL;
}
sec->idx = elf_ndxscn(s);
sec->len = size;
sec->changed = true;
sec->data = elf_newdata(s);
if (!sec->data) {
WARN_ELF("elf_newdata");
return NULL;
}
sec->data->d_size = size;
sec->data->d_align = 1;
if (size) {
sec->data->d_buf = malloc(size);
if (!sec->data->d_buf) {
perror("malloc");
return NULL;
}
memset(sec->data->d_buf, 0, size);
}
if (!gelf_getshdr(s, &sec->sh)) {
WARN_ELF("gelf_getshdr");
return NULL;
}
sec->sh.sh_size = size;
sec->sh.sh_entsize = entsize;
sec->sh.sh_type = SHT_PROGBITS;
sec->sh.sh_addralign = 1;
sec->sh.sh_flags = SHF_ALLOC;
/* Add section name to .shstrtab */
shstrtab = find_section_by_name(elf, ".shstrtab");
if (!shstrtab) {
WARN("can't find .shstrtab section");
return NULL;
}
s = elf_getscn(elf->elf, shstrtab->idx);
if (!s) {
WARN_ELF("elf_getscn");
return NULL;
}
data = elf_newdata(s);
if (!data) {
WARN_ELF("elf_newdata");
return NULL;
}
data->d_buf = sec->name;
data->d_size = strlen(name) + 1;
data->d_align = 1;
sec->sh.sh_name = shstrtab->len;
shstrtab->len += strlen(name) + 1;
shstrtab->changed = true;
return sec;
}
struct section *elf_create_rela_section(struct elf *elf, struct section *base)
{
char *relaname;
struct section *sec;
relaname = malloc(strlen(base->name) + strlen(".rela") + 1);
if (!relaname) {
perror("malloc");
return NULL;
}
strcpy(relaname, ".rela");
strcat(relaname, base->name);
sec = elf_create_section(elf, relaname, sizeof(GElf_Rela), 0);
if (!sec)
return NULL;
base->rela = sec;
sec->base = base;
sec->sh.sh_type = SHT_RELA;
sec->sh.sh_addralign = 8;
sec->sh.sh_link = find_section_by_name(elf, ".symtab")->idx;
sec->sh.sh_info = base->idx;
sec->sh.sh_flags = SHF_INFO_LINK;
return sec;
}
int elf_rebuild_rela_section(struct section *sec)
{
struct rela *rela;
int nr, idx = 0, size;
GElf_Rela *relas;
nr = 0;
list_for_each_entry(rela, &sec->rela_list, list)
nr++;
size = nr * sizeof(*relas);
relas = malloc(size);
if (!relas) {
perror("malloc");
return -1;
}
sec->data->d_buf = relas;
sec->data->d_size = size;
sec->sh.sh_size = size;
idx = 0;
list_for_each_entry(rela, &sec->rela_list, list) {
relas[idx].r_offset = rela->offset;
relas[idx].r_addend = rela->addend;
relas[idx].r_info = GELF_R_INFO(rela->sym->idx, rela->type);
idx++;
}
return 0;
}
int elf_write(struct elf *elf)
{
struct section *sec;
Elf_Scn *s;
list_for_each_entry(sec, &elf->sections, list) {
if (sec->changed) {
s = elf_getscn(elf->elf, sec->idx);
if (!s) {
WARN_ELF("elf_getscn");
return -1;
}
if (!gelf_update_shdr (s, &sec->sh)) {
WARN_ELF("gelf_update_shdr");
return -1;
}
}
}
if (elf_update(elf->elf, ELF_C_WRITE) < 0) {
WARN_ELF("elf_update");
return -1;
}
return 0;
}
void elf_close(struct elf *elf)
{
struct section *sec, *tmpsec;

15
tools/objtool/elf.h
View File

@ -28,6 +28,13 @@
# define elf_getshdrstrndx elf_getshstrndx
#endif
/*
* Fallback for systems without this "read, mmaping if possible" cmd.
*/
#ifndef ELF_C_READ_MMAP
#define ELF_C_READ_MMAP ELF_C_READ
#endif
struct section {
struct list_head list;
GElf_Shdr sh;
@ -41,6 +48,7 @@ struct section {
char *name;
int idx;
unsigned int len;
bool changed, text;
};
struct symbol {
@ -75,7 +83,7 @@ struct elf {
};
struct elf *elf_open(const char *name);
struct elf *elf_open(const char *name, int flags);
struct section *find_section_by_name(struct elf *elf, const char *name);
struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset);
struct symbol *find_symbol_containing(struct section *sec, unsigned long offset);
@ -83,6 +91,11 @@ struct rela *find_rela_by_dest(struct section *sec, unsigned long offset);
struct rela *find_rela_by_dest_range(struct section *sec, unsigned long offset,
unsigned int len);
struct symbol *find_containing_func(struct section *sec, unsigned long offset);
struct section *elf_create_section(struct elf *elf, const char *name, size_t
entsize, int nr);
struct section *elf_create_rela_section(struct elf *elf, struct section *base);
int elf_rebuild_rela_section(struct section *sec);
int elf_write(struct elf *elf);
void elf_close(struct elf *elf);
#define for_each_sec(file, sec) \

3
tools/objtool/objtool.c
View File

@ -42,10 +42,11 @@ struct cmd_struct {
};
static const char objtool_usage_string[] =
"objtool [OPTIONS] COMMAND [ARGS]";
"objtool COMMAND [ARGS]";
static struct cmd_struct objtool_cmds[] = {
{"check", cmd_check, "Perform stack metadata validation on an object file" },
{"orc", cmd_orc, "Generate in-place ORC unwind tables for an object file" },
};
bool help;

30
tools/objtool/orc.h Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _ORC_H
#define _ORC_H
#include "orc_types.h"
struct objtool_file;
int create_orc(struct objtool_file *file);
int create_orc_sections(struct objtool_file *file);
int orc_dump(const char *objname);
#endif /* _ORC_H */

212
tools/objtool/orc_dump.c Normal file
View File

@ -0,0 +1,212 @@
/*
* Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#include <unistd.h>
#include "orc.h"
#include "warn.h"
static const char *reg_name(unsigned int reg)
{
switch (reg) {
case ORC_REG_PREV_SP:
return "prevsp";
case ORC_REG_DX:
return "dx";
case ORC_REG_DI:
return "di";
case ORC_REG_BP:
return "bp";
case ORC_REG_SP:
return "sp";
case ORC_REG_R10:
return "r10";
case ORC_REG_R13:
return "r13";
case ORC_REG_BP_INDIRECT:
return "bp(ind)";
case ORC_REG_SP_INDIRECT:
return "sp(ind)";
default:
return "?";
}
}
static const char *orc_type_name(unsigned int type)
{
switch (type) {
case ORC_TYPE_CALL:
return "call";
case ORC_TYPE_REGS:
return "regs";
case ORC_TYPE_REGS_IRET:
return "iret";
default:
return "?";
}
}
static void print_reg(unsigned int reg, int offset)
{
if (reg == ORC_REG_BP_INDIRECT)
printf("(bp%+d)", offset);
else if (reg == ORC_REG_SP_INDIRECT)
printf("(sp%+d)", offset);
else if (reg == ORC_REG_UNDEFINED)
printf("(und)");
else
printf("%s%+d", reg_name(reg), offset);
}
int orc_dump(const char *_objname)
{
int fd, nr_entries, i, *orc_ip = NULL, orc_size = 0;
struct orc_entry *orc = NULL;
char *name;
unsigned long nr_sections, orc_ip_addr = 0;
size_t shstrtab_idx;
Elf *elf;
Elf_Scn *scn;
GElf_Shdr sh;
GElf_Rela rela;
GElf_Sym sym;
Elf_Data *data, *symtab = NULL, *rela_orc_ip = NULL;
objname = _objname;
elf_version(EV_CURRENT);
fd = open(objname, O_RDONLY);
if (fd == -1) {
perror("open");
return -1;
}
elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
if (!elf) {
WARN_ELF("elf_begin");
return -1;
}
if (elf_getshdrnum(elf, &nr_sections)) {
WARN_ELF("elf_getshdrnum");
return -1;
}
if (elf_getshdrstrndx(elf, &shstrtab_idx)) {
WARN_ELF("elf_getshdrstrndx");
return -1;
}
for (i = 0; i < nr_sections; i++) {
scn = elf_getscn(elf, i);
if (!scn) {
WARN_ELF("elf_getscn");
return -1;
}
if (!gelf_getshdr(scn, &sh)) {
WARN_ELF("gelf_getshdr");
return -1;
}
name = elf_strptr(elf, shstrtab_idx, sh.sh_name);
if (!name) {
WARN_ELF("elf_strptr");
return -1;
}
data = elf_getdata(scn, NULL);
if (!data) {
WARN_ELF("elf_getdata");
return -1;
}
if (!strcmp(name, ".symtab")) {
symtab = data;
} else if (!strcmp(name, ".orc_unwind")) {
orc = data->d_buf;
orc_size = sh.sh_size;
} else if (!strcmp(name, ".orc_unwind_ip")) {
orc_ip = data->d_buf;
orc_ip_addr = sh.sh_addr;
} else if (!strcmp(name, ".rela.orc_unwind_ip")) {
rela_orc_ip = data;
}
}
if (!symtab || !orc || !orc_ip)
return 0;
if (orc_size % sizeof(*orc) != 0) {
WARN("bad .orc_unwind section size");
return -1;
}
nr_entries = orc_size / sizeof(*orc);
for (i = 0; i < nr_entries; i++) {
if (rela_orc_ip) {
if (!gelf_getrela(rela_orc_ip, i, &rela)) {
WARN_ELF("gelf_getrela");
return -1;
}
if (!gelf_getsym(symtab, GELF_R_SYM(rela.r_info), &sym)) {
WARN_ELF("gelf_getsym");
return -1;
}
scn = elf_getscn(elf, sym.st_shndx);
if (!scn) {
WARN_ELF("elf_getscn");
return -1;
}
if (!gelf_getshdr(scn, &sh)) {
WARN_ELF("gelf_getshdr");
return -1;
}
name = elf_strptr(elf, shstrtab_idx, sh.sh_name);
if (!name || !*name) {
WARN_ELF("elf_strptr");
return -1;
}
printf("%s+%lx:", name, rela.r_addend);
} else {
printf("%lx:", orc_ip_addr + (i * sizeof(int)) + orc_ip[i]);
}
printf(" sp:");
print_reg(orc[i].sp_reg, orc[i].sp_offset);
printf(" bp:");
print_reg(orc[i].bp_reg, orc[i].bp_offset);
printf(" type:%s\n", orc_type_name(orc[i].type));
}
elf_end(elf);
close(fd);
return 0;
}

214
tools/objtool/orc_gen.c Normal file
View File

@ -0,0 +1,214 @@
/*
* Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#include <stdlib.h>
#include <string.h>
#include "orc.h"
#include "check.h"
#include "warn.h"
int create_orc(struct objtool_file *file)
{
struct instruction *insn;
for_each_insn(file, insn) {
struct orc_entry *orc = &insn->orc;
struct cfi_reg *cfa = &insn->state.cfa;
struct cfi_reg *bp = &insn->state.regs[CFI_BP];
if (cfa->base == CFI_UNDEFINED) {
orc->sp_reg = ORC_REG_UNDEFINED;
continue;
}
switch (cfa->base) {
case CFI_SP:
orc->sp_reg = ORC_REG_SP;
break;
case CFI_SP_INDIRECT:
orc->sp_reg = ORC_REG_SP_INDIRECT;
break;
case CFI_BP:
orc->sp_reg = ORC_REG_BP;
break;
case CFI_BP_INDIRECT:
orc->sp_reg = ORC_REG_BP_INDIRECT;
break;
case CFI_R10:
orc->sp_reg = ORC_REG_R10;
break;
case CFI_R13:
orc->sp_reg = ORC_REG_R13;
break;
case CFI_DI:
orc->sp_reg = ORC_REG_DI;
break;
case CFI_DX:
orc->sp_reg = ORC_REG_DX;
break;
default:
WARN_FUNC("unknown CFA base reg %d",
insn->sec, insn->offset, cfa->base);
return -1;
}
switch(bp->base) {
case CFI_UNDEFINED:
orc->bp_reg = ORC_REG_UNDEFINED;
break;
case CFI_CFA:
orc->bp_reg = ORC_REG_PREV_SP;
break;
case CFI_BP:
orc->bp_reg = ORC_REG_BP;
break;
default:
WARN_FUNC("unknown BP base reg %d",
insn->sec, insn->offset, bp->base);
return -1;
}
orc->sp_offset = cfa->offset;
orc->bp_offset = bp->offset;
orc->type = insn->state.type;
}
return 0;
}
static int create_orc_entry(struct section *u_sec, struct section *ip_relasec,
unsigned int idx, struct section *insn_sec,
unsigned long insn_off, struct orc_entry *o)
{
struct orc_entry *orc;
struct rela *rela;
/* populate ORC data */
orc = (struct orc_entry *)u_sec->data->d_buf + idx;
memcpy(orc, o, sizeof(*orc));
/* populate rela for ip */
rela = malloc(sizeof(*rela));
if (!rela) {
perror("malloc");
return -1;
}
memset(rela, 0, sizeof(*rela));
rela->sym = insn_sec->sym;
rela->addend = insn_off;
rela->type = R_X86_64_PC32;
rela->offset = idx * sizeof(int);
list_add_tail(&rela->list, &ip_relasec->rela_list);
hash_add(ip_relasec->rela_hash, &rela->hash, rela->offset);
return 0;
}
int create_orc_sections(struct objtool_file *file)
{
struct instruction *insn, *prev_insn;
struct section *sec, *u_sec, *ip_relasec;
unsigned int idx;
struct orc_entry empty = {
.sp_reg = ORC_REG_UNDEFINED,
.bp_reg = ORC_REG_UNDEFINED,
.type = ORC_TYPE_CALL,
};
sec = find_section_by_name(file->elf, ".orc_unwind");
if (sec) {
WARN("file already has .orc_unwind section, skipping");
return -1;
}
/* count the number of needed orcs */
idx = 0;
for_each_sec(file, sec) {
if (!sec->text)
continue;
prev_insn = NULL;
sec_for_each_insn(file, sec, insn) {
if (!prev_insn ||
memcmp(&insn->orc, &prev_insn->orc,
sizeof(struct orc_entry))) {
idx++;
}
prev_insn = insn;
}
/* section terminator */
if (prev_insn)
idx++;
}
if (!idx)
return -1;
/* create .orc_unwind_ip and .rela.orc_unwind_ip sections */
sec = elf_create_section(file->elf, ".orc_unwind_ip", sizeof(int), idx);
ip_relasec = elf_create_rela_section(file->elf, sec);
if (!ip_relasec)
return -1;
/* create .orc_unwind section */
u_sec = elf_create_section(file->elf, ".orc_unwind",
sizeof(struct orc_entry), idx);
/* populate sections */
idx = 0;
for_each_sec(file, sec) {
if (!sec->text)
continue;
prev_insn = NULL;
sec_for_each_insn(file, sec, insn) {
if (!prev_insn || memcmp(&insn->orc, &prev_insn->orc,
sizeof(struct orc_entry))) {
if (create_orc_entry(u_sec, ip_relasec, idx,
insn->sec, insn->offset,
&insn->orc))
return -1;
idx++;
}
prev_insn = insn;
}
/* section terminator */
if (prev_insn) {
if (create_orc_entry(u_sec, ip_relasec, idx,
prev_insn->sec,
prev_insn->offset + prev_insn->len,
&empty))
return -1;
idx++;
}
}
if (elf_rebuild_rela_section(ip_relasec))
return -1;
return 0;
}

107
tools/objtool/orc_types.h Normal file
View File

@ -0,0 +1,107 @@
/*
* Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _ORC_TYPES_H
#define _ORC_TYPES_H
#include <linux/types.h>
#include <linux/compiler.h>
/*
* The ORC_REG_* registers are base registers which are used to find other
* registers on the stack.
*
* ORC_REG_PREV_SP, also known as DWARF Call Frame Address (CFA), is the
* address of the previous frame: the caller's SP before it called the current
* function.
*
* ORC_REG_UNDEFINED means the corresponding register's value didn't change in
* the current frame.
*
* The most commonly used base registers are SP and BP -- which the previous SP
* is usually based on -- and PREV_SP and UNDEFINED -- which the previous BP is
* usually based on.
*
* The rest of the base registers are needed for special cases like entry code
* and GCC realigned stacks.
*/
#define ORC_REG_UNDEFINED 0
#define ORC_REG_PREV_SP 1
#define ORC_REG_DX 2
#define ORC_REG_DI 3
#define ORC_REG_BP 4
#define ORC_REG_SP 5
#define ORC_REG_R10 6
#define ORC_REG_R13 7
#define ORC_REG_BP_INDIRECT 8
#define ORC_REG_SP_INDIRECT 9
#define ORC_REG_MAX 15
/*
* ORC_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP (the
* caller's SP right before it made the call). Used for all callable
* functions, i.e. all C code and all callable asm functions.
*
* ORC_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset points
* to a fully populated pt_regs from a syscall, interrupt, or exception.
*
* ORC_TYPE_REGS_IRET: Used in entry code to indicate that sp_reg+sp_offset
* points to the iret return frame.
*
* The UNWIND_HINT macros are used only for the unwind_hint struct. They
* aren't used in struct orc_entry due to size and complexity constraints.
* Objtool converts them to real types when it converts the hints to orc
* entries.
*/
#define ORC_TYPE_CALL 0
#define ORC_TYPE_REGS 1
#define ORC_TYPE_REGS_IRET 2
#define UNWIND_HINT_TYPE_SAVE 3
#define UNWIND_HINT_TYPE_RESTORE 4
#ifndef __ASSEMBLY__
/*
* This struct is more or less a vastly simplified version of the DWARF Call
* Frame Information standard. It contains only the necessary parts of DWARF
* CFI, simplified for ease of access by the in-kernel unwinder. It tells the
* unwinder how to find the previous SP and BP (and sometimes entry regs) on
* the stack for a given code address. Each instance of the struct corresponds
* to one or more code locations.
*/
struct orc_entry {
s16 sp_offset;
s16 bp_offset;
unsigned sp_reg:4;
unsigned bp_reg:4;
unsigned type:2;
} __packed;
/*
* This struct is used by asm and inline asm code to manually annotate the
* location of registers on the stack for the ORC unwinder.
*
* Type can be either ORC_TYPE_* or UNWIND_HINT_TYPE_*.
*/
struct unwind_hint {
u32 ip;
s16 sp_offset;
u8 sp_reg;
u8 type;
};
#endif /* __ASSEMBLY__ */
#endif /* _ORC_TYPES_H */