mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2025-01-22 22:04:47 +08:00
rseq/selftests: Provide rseq library
This rseq helper library provides a user-space API to the rseq() system call. The rseq fast-path exposes the instruction pointer addresses where the rseq assembly blocks begin and end, as well as the associated abort instruction pointer, in the __rseq_table section. This section allows debuggers may know where to place breakpoints when single-stepping through assembly blocks which may be aborted at any point by the kernel. Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com> Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Cc: Joel Fernandes <joelaf@google.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Dave Watson <davejwatson@fb.com> Cc: Will Deacon <will.deacon@arm.com> Cc: Shuah Khan <shuahkh@osg.samsung.com> Cc: Andi Kleen <andi@firstfloor.org> Cc: linux-kselftest@vger.kernel.org Cc: "H . Peter Anvin" <hpa@zytor.com> Cc: Chris Lameter <cl@linux.com> Cc: Russell King <linux@arm.linux.org.uk> Cc: Andrew Hunter <ahh@google.com> Cc: Michael Kerrisk <mtk.manpages@gmail.com> Cc: "Paul E . McKenney" <paulmck@linux.vnet.ibm.com> Cc: Paul Turner <pjt@google.com> Cc: Boqun Feng <boqun.feng@gmail.com> Cc: Josh Triplett <josh@joshtriplett.org> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Ben Maurer <bmaurer@fb.com> Cc: linux-api@vger.kernel.org Cc: Andy Lutomirski <luto@amacapital.net> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Link: https://lkml.kernel.org/r/20180602124408.8430-13-mathieu.desnoyers@efficios.com
This commit is contained in:
Mathieu Desnoyers
Thomas Gleixner
parent
4e49ed2f9a
commit
2e155fb7d6
715
tools/testing/selftests/rseq/rseq-arm.h
Normal file
715
tools/testing/selftests/rseq/rseq-arm.h
Normal file
@ -0,0 +1,715 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
|
||||
/*
|
||||
* rseq-arm.h
|
||||
*
|
||||
* (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*/
|
||||
|
||||
#define RSEQ_SIG 0x53053053
|
||||
|
||||
#define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
|
||||
#define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
|
||||
#define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
|
||||
|
||||
#define rseq_smp_load_acquire(p) \
|
||||
__extension__ ({ \
|
||||
__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
|
||||
rseq_smp_mb(); \
|
||||
____p1; \
|
||||
})
|
||||
|
||||
#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
|
||||
|
||||
#define rseq_smp_store_release(p, v) \
|
||||
do { \
|
||||
rseq_smp_mb(); \
|
||||
RSEQ_WRITE_ONCE(*p, v); \
|
||||
} while (0)
|
||||
|
||||
#ifdef RSEQ_SKIP_FASTPATH
|
||||
#include "rseq-skip.h"
|
||||
#else /* !RSEQ_SKIP_FASTPATH */
|
||||
|
||||
#define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \
|
||||
post_commit_offset, abort_ip) \
|
||||
".pushsection __rseq_table, \"aw\"\n\t" \
|
||||
".balign 32\n\t" \
|
||||
".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
|
||||
".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
|
||||
".popsection\n\t"
|
||||
|
||||
#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \
|
||||
__RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \
|
||||
(post_commit_ip - start_ip), abort_ip)
|
||||
|
||||
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
|
||||
RSEQ_INJECT_ASM(1) \
|
||||
"adr r0, " __rseq_str(cs_label) "\n\t" \
|
||||
"str r0, %[" __rseq_str(rseq_cs) "]\n\t" \
|
||||
__rseq_str(label) ":\n\t"
|
||||
|
||||
#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
|
||||
RSEQ_INJECT_ASM(2) \
|
||||
"ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
|
||||
"cmp %[" __rseq_str(cpu_id) "], r0\n\t" \
|
||||
"bne " __rseq_str(label) "\n\t"
|
||||
|
||||
#define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
|
||||
abort_label, version, flags, \
|
||||
start_ip, post_commit_offset, abort_ip) \
|
||||
__rseq_str(table_label) ":\n\t" \
|
||||
".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
|
||||
".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
|
||||
".word " __rseq_str(RSEQ_SIG) "\n\t" \
|
||||
__rseq_str(label) ":\n\t" \
|
||||
teardown \
|
||||
"b %l[" __rseq_str(abort_label) "]\n\t"
|
||||
|
||||
#define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
|
||||
start_ip, post_commit_ip, abort_ip) \
|
||||
__RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
|
||||
abort_label, 0x0, 0x0, start_ip, \
|
||||
(post_commit_ip - start_ip), abort_ip)
|
||||
|
||||
#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
|
||||
__rseq_str(label) ":\n\t" \
|
||||
teardown \
|
||||
"b %l[" __rseq_str(cmpfail_label) "]\n\t"
|
||||
|
||||
#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne %l[error2]\n\t"
|
||||
#endif
|
||||
/* final store */
|
||||
"str %[newv], %[v]\n\t"
|
||||
"2:\n\t"
|
||||
RSEQ_INJECT_ASM(5)
|
||||
"b 5f\n\t"
|
||||
RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "r0", "memory", "cc"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expectnot], r0\n\t"
|
||||
"beq %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expectnot], r0\n\t"
|
||||
"beq %l[error2]\n\t"
|
||||
#endif
|
||||
"str r0, %[load]\n\t"
|
||||
"add r0, %[voffp]\n\t"
|
||||
"ldr r0, [r0]\n\t"
|
||||
/* final store */
|
||||
"str r0, %[v]\n\t"
|
||||
"2:\n\t"
|
||||
RSEQ_INJECT_ASM(5)
|
||||
"b 5f\n\t"
|
||||
RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expectnot] "r" (expectnot),
|
||||
[voffp] "Ir" (voffp),
|
||||
[load] "m" (*load)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "r0", "memory", "cc"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
#endif
|
||||
"ldr r0, %[v]\n\t"
|
||||
"add r0, %[count]\n\t"
|
||||
/* final store */
|
||||
"str r0, %[v]\n\t"
|
||||
"2:\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
"b 5f\n\t"
|
||||
RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[v] "m" (*v),
|
||||
[count] "Ir" (count)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "r0", "memory", "cc"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
intptr_t *v2, intptr_t newv2,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne %l[error2]\n\t"
|
||||
#endif
|
||||
/* try store */
|
||||
"str %[newv2], %[v2]\n\t"
|
||||
RSEQ_INJECT_ASM(5)
|
||||
/* final store */
|
||||
"str %[newv], %[v]\n\t"
|
||||
"2:\n\t"
|
||||
RSEQ_INJECT_ASM(6)
|
||||
"b 5f\n\t"
|
||||
RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "r0", "memory", "cc"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
intptr_t *v2, intptr_t newv2,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne %l[error2]\n\t"
|
||||
#endif
|
||||
/* try store */
|
||||
"str %[newv2], %[v2]\n\t"
|
||||
RSEQ_INJECT_ASM(5)
|
||||
"dmb\n\t" /* full mb provides store-release */
|
||||
/* final store */
|
||||
"str %[newv], %[v]\n\t"
|
||||
"2:\n\t"
|
||||
RSEQ_INJECT_ASM(6)
|
||||
"b 5f\n\t"
|
||||
RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "r0", "memory", "cc"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
intptr_t *v2, intptr_t expect2,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
"ldr r0, %[v2]\n\t"
|
||||
"cmp %[expect2], r0\n\t"
|
||||
"bne %l[cmpfail]\n\t"
|
||||
RSEQ_INJECT_ASM(5)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne %l[error2]\n\t"
|
||||
"ldr r0, %[v2]\n\t"
|
||||
"cmp %[expect2], r0\n\t"
|
||||
"bne %l[error3]\n\t"
|
||||
#endif
|
||||
/* final store */
|
||||
"str %[newv], %[v]\n\t"
|
||||
"2:\n\t"
|
||||
RSEQ_INJECT_ASM(6)
|
||||
"b 5f\n\t"
|
||||
RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
|
||||
"5:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* cmp2 input */
|
||||
[v2] "m" (*v2),
|
||||
[expect2] "r" (expect2),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "r0", "memory", "cc"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2, error3
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("1st expected value comparison failed");
|
||||
error3:
|
||||
rseq_bug("2nd expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
void *dst, void *src, size_t len,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
uint32_t rseq_scratch[3];
|
||||
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
|
||||
"str %[src], %[rseq_scratch0]\n\t"
|
||||
"str %[dst], %[rseq_scratch1]\n\t"
|
||||
"str %[len], %[rseq_scratch2]\n\t"
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne 5f\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne 7f\n\t"
|
||||
#endif
|
||||
/* try memcpy */
|
||||
"cmp %[len], #0\n\t" \
|
||||
"beq 333f\n\t" \
|
||||
"222:\n\t" \
|
||||
"ldrb %%r0, [%[src]]\n\t" \
|
||||
"strb %%r0, [%[dst]]\n\t" \
|
||||
"adds %[src], #1\n\t" \
|
||||
"adds %[dst], #1\n\t" \
|
||||
"subs %[len], #1\n\t" \
|
||||
"bne 222b\n\t" \
|
||||
"333:\n\t" \
|
||||
RSEQ_INJECT_ASM(5)
|
||||
/* final store */
|
||||
"str %[newv], %[v]\n\t"
|
||||
"2:\n\t"
|
||||
RSEQ_INJECT_ASM(6)
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t"
|
||||
"b 8f\n\t"
|
||||
RSEQ_ASM_DEFINE_ABORT(3, 4,
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t",
|
||||
abort, 1b, 2b, 4f)
|
||||
RSEQ_ASM_DEFINE_CMPFAIL(5,
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t",
|
||||
cmpfail)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_DEFINE_CMPFAIL(6,
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t",
|
||||
error1)
|
||||
RSEQ_ASM_DEFINE_CMPFAIL(7,
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t",
|
||||
error2)
|
||||
#endif
|
||||
"8:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv),
|
||||
/* try memcpy input */
|
||||
[dst] "r" (dst),
|
||||
[src] "r" (src),
|
||||
[len] "r" (len),
|
||||
[rseq_scratch0] "m" (rseq_scratch[0]),
|
||||
[rseq_scratch1] "m" (rseq_scratch[1]),
|
||||
[rseq_scratch2] "m" (rseq_scratch[2])
|
||||
RSEQ_INJECT_INPUT
|
||||
: "r0", "memory", "cc"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
void *dst, void *src, size_t len,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
uint32_t rseq_scratch[3];
|
||||
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
|
||||
"str %[src], %[rseq_scratch0]\n\t"
|
||||
"str %[dst], %[rseq_scratch1]\n\t"
|
||||
"str %[len], %[rseq_scratch2]\n\t"
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne 5f\n\t"
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
|
||||
"ldr r0, %[v]\n\t"
|
||||
"cmp %[expect], r0\n\t"
|
||||
"bne 7f\n\t"
|
||||
#endif
|
||||
/* try memcpy */
|
||||
"cmp %[len], #0\n\t" \
|
||||
"beq 333f\n\t" \
|
||||
"222:\n\t" \
|
||||
"ldrb %%r0, [%[src]]\n\t" \
|
||||
"strb %%r0, [%[dst]]\n\t" \
|
||||
"adds %[src], #1\n\t" \
|
||||
"adds %[dst], #1\n\t" \
|
||||
"subs %[len], #1\n\t" \
|
||||
"bne 222b\n\t" \
|
||||
"333:\n\t" \
|
||||
RSEQ_INJECT_ASM(5)
|
||||
"dmb\n\t" /* full mb provides store-release */
|
||||
/* final store */
|
||||
"str %[newv], %[v]\n\t"
|
||||
"2:\n\t"
|
||||
RSEQ_INJECT_ASM(6)
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t"
|
||||
"b 8f\n\t"
|
||||
RSEQ_ASM_DEFINE_ABORT(3, 4,
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t",
|
||||
abort, 1b, 2b, 4f)
|
||||
RSEQ_ASM_DEFINE_CMPFAIL(5,
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t",
|
||||
cmpfail)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
RSEQ_ASM_DEFINE_CMPFAIL(6,
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t",
|
||||
error1)
|
||||
RSEQ_ASM_DEFINE_CMPFAIL(7,
|
||||
/* teardown */
|
||||
"ldr %[len], %[rseq_scratch2]\n\t"
|
||||
"ldr %[dst], %[rseq_scratch1]\n\t"
|
||||
"ldr %[src], %[rseq_scratch0]\n\t",
|
||||
error2)
|
||||
#endif
|
||||
"8:\n\t"
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv),
|
||||
/* try memcpy input */
|
||||
[dst] "r" (dst),
|
||||
[src] "r" (src),
|
||||
[len] "r" (len),
|
||||
[rseq_scratch0] "m" (rseq_scratch[0]),
|
||||
[rseq_scratch1] "m" (rseq_scratch[1]),
|
||||
[rseq_scratch2] "m" (rseq_scratch[2])
|
||||
RSEQ_INJECT_INPUT
|
||||
: "r0", "memory", "cc"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 0;
|
||||
abort:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_workaround_gcc_asm_size_guess();
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* !RSEQ_SKIP_FASTPATH */
|
||||
671
tools/testing/selftests/rseq/rseq-ppc.h
Normal file
671
tools/testing/selftests/rseq/rseq-ppc.h
Normal file
@ -0,0 +1,671 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
|
||||
/*
|
||||
* rseq-ppc.h
|
||||
*
|
||||
* (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
* (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
|
||||
*/
|
||||
|
||||
#define RSEQ_SIG 0x53053053
|
||||
|
||||
#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
|
||||
#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
|
||||
#define rseq_smp_rmb() rseq_smp_lwsync()
|
||||
#define rseq_smp_wmb() rseq_smp_lwsync()
|
||||
|
||||
#define rseq_smp_load_acquire(p) \
|
||||
__extension__ ({ \
|
||||
__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
|
||||
rseq_smp_lwsync(); \
|
||||
____p1; \
|
||||
})
|
||||
|
||||
#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
|
||||
|
||||
#define rseq_smp_store_release(p, v) \
|
||||
do { \
|
||||
rseq_smp_lwsync(); \
|
||||
RSEQ_WRITE_ONCE(*p, v); \
|
||||
} while (0)
|
||||
|
||||
#ifdef RSEQ_SKIP_FASTPATH
|
||||
#include "rseq-skip.h"
|
||||
#else /* !RSEQ_SKIP_FASTPATH */
|
||||
|
||||
/*
|
||||
* The __rseq_table section can be used by debuggers to better handle
|
||||
* single-stepping through the restartable critical sections.
|
||||
*/
|
||||
|
||||
#ifdef __PPC64__
|
||||
|
||||
#define STORE_WORD "std "
|
||||
#define LOAD_WORD "ld "
|
||||
#define LOADX_WORD "ldx "
|
||||
#define CMP_WORD "cmpd "
|
||||
|
||||
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
|
||||
start_ip, post_commit_offset, abort_ip) \
|
||||
".pushsection __rseq_table, \"aw\"\n\t" \
|
||||
".balign 32\n\t" \
|
||||
__rseq_str(label) ":\n\t" \
|
||||
".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
|
||||
".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
|
||||
".popsection\n\t"
|
||||
|
||||
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
|
||||
RSEQ_INJECT_ASM(1) \
|
||||
"lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
|
||||
"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
|
||||
"rldicr %%r17, %%r17, 32, 31\n\t" \
|
||||
"oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
|
||||
"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
|
||||
"std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
|
||||
__rseq_str(label) ":\n\t"
|
||||
|
||||
#else /* #ifdef __PPC64__ */
|
||||
|
||||
#define STORE_WORD "stw "
|
||||
#define LOAD_WORD "lwz "
|
||||
#define LOADX_WORD "lwzx "
|
||||
#define CMP_WORD "cmpw "
|
||||
|
||||
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
|
||||
start_ip, post_commit_offset, abort_ip) \
|
||||
".pushsection __rseq_table, \"aw\"\n\t" \
|
||||
".balign 32\n\t" \
|
||||
__rseq_str(label) ":\n\t" \
|
||||
".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
|
||||
/* 32-bit only supported on BE */ \
|
||||
".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
|
||||
".popsection\n\t"
|
||||
|
||||
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
|
||||
RSEQ_INJECT_ASM(1) \
|
||||
"lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
|
||||
"addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
|
||||
"stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
|
||||
__rseq_str(label) ":\n\t"
|
||||
|
||||
#endif /* #ifdef __PPC64__ */
|
||||
|
||||
#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
|
||||
__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
|
||||
(post_commit_ip - start_ip), abort_ip)
|
||||
|
||||
#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
|
||||
RSEQ_INJECT_ASM(2) \
|
||||
"lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
|
||||
"cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
|
||||
"bne- cr7, " __rseq_str(label) "\n\t"
|
||||
|
||||
#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
|
||||
".pushsection __rseq_failure, \"ax\"\n\t" \
|
||||
".long " __rseq_str(RSEQ_SIG) "\n\t" \
|
||||
__rseq_str(label) ":\n\t" \
|
||||
"b %l[" __rseq_str(abort_label) "]\n\t" \
|
||||
".popsection\n\t"
|
||||
|
||||
/*
|
||||
* RSEQ_ASM_OPs: asm operations for rseq
|
||||
* RSEQ_ASM_OP_R_*: has hard-code registers in it
|
||||
* RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
|
||||
*/
|
||||
#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
|
||||
LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
|
||||
CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
|
||||
"bne- cr7, " __rseq_str(label) "\n\t"
|
||||
|
||||
#define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
|
||||
LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
|
||||
CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
|
||||
"beq- cr7, " __rseq_str(label) "\n\t"
|
||||
|
||||
#define RSEQ_ASM_OP_STORE(value, var) \
|
||||
STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
|
||||
|
||||
/* Load @var to r17 */
|
||||
#define RSEQ_ASM_OP_R_LOAD(var) \
|
||||
LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
|
||||
|
||||
/* Store r17 to @var */
|
||||
#define RSEQ_ASM_OP_R_STORE(var) \
|
||||
STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
|
||||
|
||||
/* Add @count to r17 */
|
||||
#define RSEQ_ASM_OP_R_ADD(count) \
|
||||
"add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
|
||||
|
||||
/* Load (r17 + voffp) to r17 */
|
||||
#define RSEQ_ASM_OP_R_LOADX(voffp) \
|
||||
LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
|
||||
|
||||
/* TODO: implement a faster memcpy. */
|
||||
#define RSEQ_ASM_OP_R_MEMCPY() \
|
||||
"cmpdi %%r19, 0\n\t" \
|
||||
"beq 333f\n\t" \
|
||||
"addi %%r20, %%r20, -1\n\t" \
|
||||
"addi %%r21, %%r21, -1\n\t" \
|
||||
"222:\n\t" \
|
||||
"lbzu %%r18, 1(%%r20)\n\t" \
|
||||
"stbu %%r18, 1(%%r21)\n\t" \
|
||||
"addi %%r19, %%r19, -1\n\t" \
|
||||
"cmpdi %%r19, 0\n\t" \
|
||||
"bne 222b\n\t" \
|
||||
"333:\n\t" \
|
||||
|
||||
#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
|
||||
STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
|
||||
__rseq_str(post_commit_label) ":\n\t"
|
||||
|
||||
#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
|
||||
STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
|
||||
__rseq_str(post_commit_label) ":\n\t"
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
|
||||
#endif
|
||||
/* final store */
|
||||
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
|
||||
RSEQ_INJECT_ASM(5)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "memory", "cc", "r17"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
return 0;
|
||||
abort:
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
/* cmp @v not equal to @expectnot */
|
||||
RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
/* cmp @v not equal to @expectnot */
|
||||
RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
|
||||
#endif
|
||||
/* load the value of @v */
|
||||
RSEQ_ASM_OP_R_LOAD(v)
|
||||
/* store it in @load */
|
||||
RSEQ_ASM_OP_R_STORE(load)
|
||||
/* dereference voffp(v) */
|
||||
RSEQ_ASM_OP_R_LOADX(voffp)
|
||||
/* final store the value at voffp(v) */
|
||||
RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
|
||||
RSEQ_INJECT_ASM(5)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expectnot] "r" (expectnot),
|
||||
[voffp] "b" (voffp),
|
||||
[load] "m" (*load)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "memory", "cc", "r17"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
return 0;
|
||||
abort:
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
#endif
|
||||
/* load the value of @v */
|
||||
RSEQ_ASM_OP_R_LOAD(v)
|
||||
/* add @count to it */
|
||||
RSEQ_ASM_OP_R_ADD(count)
|
||||
/* final store */
|
||||
RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
|
||||
RSEQ_INJECT_ASM(4)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[count] "r" (count)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "memory", "cc", "r17"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1
|
||||
#endif
|
||||
);
|
||||
return 0;
|
||||
abort:
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
intptr_t *v2, intptr_t newv2,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
|
||||
#endif
|
||||
/* try store */
|
||||
RSEQ_ASM_OP_STORE(newv2, v2)
|
||||
RSEQ_INJECT_ASM(5)
|
||||
/* final store */
|
||||
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
|
||||
RSEQ_INJECT_ASM(6)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "memory", "cc", "r17"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
return 0;
|
||||
abort:
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
intptr_t *v2, intptr_t newv2,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
|
||||
#endif
|
||||
/* try store */
|
||||
RSEQ_ASM_OP_STORE(newv2, v2)
|
||||
RSEQ_INJECT_ASM(5)
|
||||
/* for 'release' */
|
||||
"lwsync\n\t"
|
||||
/* final store */
|
||||
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
|
||||
RSEQ_INJECT_ASM(6)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* try store input */
|
||||
[v2] "m" (*v2),
|
||||
[newv2] "r" (newv2),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "memory", "cc", "r17"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
return 0;
|
||||
abort:
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
intptr_t *v2, intptr_t expect2,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
|
||||
RSEQ_INJECT_ASM(4)
|
||||
/* cmp @v2 equal to @expct2 */
|
||||
RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
|
||||
RSEQ_INJECT_ASM(5)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
|
||||
/* cmp @v2 equal to @expct2 */
|
||||
RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
|
||||
#endif
|
||||
/* final store */
|
||||
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
|
||||
RSEQ_INJECT_ASM(6)
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* cmp2 input */
|
||||
[v2] "m" (*v2),
|
||||
[expect2] "r" (expect2),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "memory", "cc", "r17"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2, error3
|
||||
#endif
|
||||
);
|
||||
return 0;
|
||||
abort:
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("1st expected value comparison failed");
|
||||
error3:
|
||||
rseq_bug("2nd expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
void *dst, void *src, size_t len,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
|
||||
/* setup for mempcy */
|
||||
"mr %%r19, %[len]\n\t"
|
||||
"mr %%r20, %[src]\n\t"
|
||||
"mr %%r21, %[dst]\n\t"
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
|
||||
#endif
|
||||
/* try memcpy */
|
||||
RSEQ_ASM_OP_R_MEMCPY()
|
||||
RSEQ_INJECT_ASM(5)
|
||||
/* final store */
|
||||
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
|
||||
RSEQ_INJECT_ASM(6)
|
||||
/* teardown */
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv),
|
||||
/* try memcpy input */
|
||||
[dst] "r" (dst),
|
||||
[src] "r" (src),
|
||||
[len] "r" (len)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
return 0;
|
||||
abort:
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
void *dst, void *src, size_t len,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
RSEQ_INJECT_C(9)
|
||||
|
||||
__asm__ __volatile__ goto (
|
||||
RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
|
||||
/* setup for mempcy */
|
||||
"mr %%r19, %[len]\n\t"
|
||||
"mr %%r20, %[src]\n\t"
|
||||
"mr %%r21, %[dst]\n\t"
|
||||
/* Start rseq by storing table entry pointer into rseq_cs. */
|
||||
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
|
||||
RSEQ_INJECT_ASM(3)
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
|
||||
RSEQ_INJECT_ASM(4)
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
/* cmp cpuid */
|
||||
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
|
||||
/* cmp @v equal to @expect */
|
||||
RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
|
||||
#endif
|
||||
/* try memcpy */
|
||||
RSEQ_ASM_OP_R_MEMCPY()
|
||||
RSEQ_INJECT_ASM(5)
|
||||
/* for 'release' */
|
||||
"lwsync\n\t"
|
||||
/* final store */
|
||||
RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
|
||||
RSEQ_INJECT_ASM(6)
|
||||
/* teardown */
|
||||
RSEQ_ASM_DEFINE_ABORT(4, abort)
|
||||
: /* gcc asm goto does not allow outputs */
|
||||
: [cpu_id] "r" (cpu),
|
||||
[current_cpu_id] "m" (__rseq_abi.cpu_id),
|
||||
[rseq_cs] "m" (__rseq_abi.rseq_cs),
|
||||
/* final store input */
|
||||
[v] "m" (*v),
|
||||
[expect] "r" (expect),
|
||||
[newv] "r" (newv),
|
||||
/* try memcpy input */
|
||||
[dst] "r" (dst),
|
||||
[src] "r" (src),
|
||||
[len] "r" (len)
|
||||
RSEQ_INJECT_INPUT
|
||||
: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
|
||||
RSEQ_INJECT_CLOBBER
|
||||
: abort, cmpfail
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
, error1, error2
|
||||
#endif
|
||||
);
|
||||
return 0;
|
||||
abort:
|
||||
RSEQ_INJECT_FAILED
|
||||
return -1;
|
||||
cmpfail:
|
||||
return 1;
|
||||
#ifdef RSEQ_COMPARE_TWICE
|
||||
error1:
|
||||
rseq_bug("cpu_id comparison failed");
|
||||
error2:
|
||||
rseq_bug("expected value comparison failed");
|
||||
#endif
|
||||
}
|
||||
|
||||
#undef STORE_WORD
|
||||
#undef LOAD_WORD
|
||||
#undef LOADX_WORD
|
||||
#undef CMP_WORD
|
||||
|
||||
#endif /* !RSEQ_SKIP_FASTPATH */
|
||||
65
tools/testing/selftests/rseq/rseq-skip.h
Normal file
65
tools/testing/selftests/rseq/rseq-skip.h
Normal file
@ -0,0 +1,65 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
|
||||
/*
|
||||
* rseq-skip.h
|
||||
*
|
||||
* (C) Copyright 2017-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*/
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
|
||||
off_t voffp, intptr_t *load, int cpu)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_addv(intptr_t *v, intptr_t count, int cpu)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
|
||||
intptr_t *v2, intptr_t newv2,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
|
||||
intptr_t *v2, intptr_t newv2,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
|
||||
intptr_t *v2, intptr_t expect2,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
|
||||
void *dst, void *src, size_t len,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
static inline __attribute__((always_inline))
|
||||
int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
|
||||
void *dst, void *src, size_t len,
|
||||
intptr_t newv, int cpu)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
1132
tools/testing/selftests/rseq/rseq-x86.h
Normal file
1132
tools/testing/selftests/rseq/rseq-x86.h
Normal file
File diff suppressed because it is too large
Load Diff
117
tools/testing/selftests/rseq/rseq.c
Normal file
117
tools/testing/selftests/rseq/rseq.c
Normal file
@ -0,0 +1,117 @@
|
||||
// SPDX-License-Identifier: LGPL-2.1
|
||||
/*
|
||||
* rseq.c
|
||||
*
|
||||
* Copyright (C) 2016 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; only
|
||||
* version 2.1 of the License.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*/
|
||||
|
||||
#define _GNU_SOURCE
|
||||
#include <errno.h>
|
||||
#include <sched.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <syscall.h>
|
||||
#include <assert.h>
|
||||
#include <signal.h>
|
||||
|
||||
#include "rseq.h"
|
||||
|
||||
#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
|
||||
|
||||
__attribute__((tls_model("initial-exec"))) __thread
|
||||
volatile struct rseq __rseq_abi = {
|
||||
.cpu_id = RSEQ_CPU_ID_UNINITIALIZED,
|
||||
};
|
||||
|
||||
static __attribute__((tls_model("initial-exec"))) __thread
|
||||
volatile int refcount;
|
||||
|
||||
static void signal_off_save(sigset_t *oldset)
|
||||
{
|
||||
sigset_t set;
|
||||
int ret;
|
||||
|
||||
sigfillset(&set);
|
||||
ret = pthread_sigmask(SIG_BLOCK, &set, oldset);
|
||||
if (ret)
|
||||
abort();
|
||||
}
|
||||
|
||||
static void signal_restore(sigset_t oldset)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = pthread_sigmask(SIG_SETMASK, &oldset, NULL);
|
||||
if (ret)
|
||||
abort();
|
||||
}
|
||||
|
||||
static int sys_rseq(volatile struct rseq *rseq_abi, uint32_t rseq_len,
|
||||
int flags, uint32_t sig)
|
||||
{
|
||||
return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig);
|
||||
}
|
||||
|
||||
int rseq_register_current_thread(void)
|
||||
{
|
||||
int rc, ret = 0;
|
||||
sigset_t oldset;
|
||||
|
||||
signal_off_save(&oldset);
|
||||
if (refcount++)
|
||||
goto end;
|
||||
rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 0, RSEQ_SIG);
|
||||
if (!rc) {
|
||||
assert(rseq_current_cpu_raw() >= 0);
|
||||
goto end;
|
||||
}
|
||||
if (errno != EBUSY)
|
||||
__rseq_abi.cpu_id = -2;
|
||||
ret = -1;
|
||||
refcount--;
|
||||
end:
|
||||
signal_restore(oldset);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int rseq_unregister_current_thread(void)
|
||||
{
|
||||
int rc, ret = 0;
|
||||
sigset_t oldset;
|
||||
|
||||
signal_off_save(&oldset);
|
||||
if (--refcount)
|
||||
goto end;
|
||||
rc = sys_rseq(&__rseq_abi, sizeof(struct rseq),
|
||||
RSEQ_FLAG_UNREGISTER, RSEQ_SIG);
|
||||
if (!rc)
|
||||
goto end;
|
||||
ret = -1;
|
||||
end:
|
||||
signal_restore(oldset);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int32_t rseq_fallback_current_cpu(void)
|
||||
{
|
||||
int32_t cpu;
|
||||
|
||||
cpu = sched_getcpu();
|
||||
if (cpu < 0) {
|
||||
perror("sched_getcpu()");
|
||||
abort();
|
||||
}
|
||||
return cpu;
|
||||
}
|
||||
147
tools/testing/selftests/rseq/rseq.h
Normal file
147
tools/testing/selftests/rseq/rseq.h
Normal file
@ -0,0 +1,147 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
|
||||
/*
|
||||
* rseq.h
|
||||
*
|
||||
* (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
|
||||
*/
|
||||
|
||||
#ifndef RSEQ_H
|
||||
#define RSEQ_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <pthread.h>
|
||||
#include <signal.h>
|
||||
#include <sched.h>
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sched.h>
|
||||
#include <linux/rseq.h>
|
||||
|
||||
/*
|
||||
* Empty code injection macros, override when testing.
|
||||
* It is important to consider that the ASM injection macros need to be
|
||||
* fully reentrant (e.g. do not modify the stack).
|
||||
*/
|
||||
#ifndef RSEQ_INJECT_ASM
|
||||
#define RSEQ_INJECT_ASM(n)
|
||||
#endif
|
||||
|
||||
#ifndef RSEQ_INJECT_C
|
||||
#define RSEQ_INJECT_C(n)
|
||||
#endif
|
||||
|
||||
#ifndef RSEQ_INJECT_INPUT
|
||||
#define RSEQ_INJECT_INPUT
|
||||
#endif
|
||||
|
||||
#ifndef RSEQ_INJECT_CLOBBER
|
||||
#define RSEQ_INJECT_CLOBBER
|
||||
#endif
|
||||
|
||||
#ifndef RSEQ_INJECT_FAILED
|
||||
#define RSEQ_INJECT_FAILED
|
||||
#endif
|
||||
|
||||
extern __thread volatile struct rseq __rseq_abi;
|
||||
|
||||
#define rseq_likely(x) __builtin_expect(!!(x), 1)
|
||||
#define rseq_unlikely(x) __builtin_expect(!!(x), 0)
|
||||
#define rseq_barrier() __asm__ __volatile__("" : : : "memory")
|
||||
|
||||
#define RSEQ_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x))
|
||||
#define RSEQ_WRITE_ONCE(x, v) __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); })
|
||||
#define RSEQ_READ_ONCE(x) RSEQ_ACCESS_ONCE(x)
|
||||
|
||||
#define __rseq_str_1(x) #x
|
||||
#define __rseq_str(x) __rseq_str_1(x)
|
||||
|
||||
#define rseq_log(fmt, args...) \
|
||||
fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \
|
||||
## args, __func__)
|
||||
|
||||
#define rseq_bug(fmt, args...) \
|
||||
do { \
|
||||
rseq_log(fmt, ##args); \
|
||||
abort(); \
|
||||
} while (0)
|
||||
|
||||
#if defined(__x86_64__) || defined(__i386__)
|
||||
#include <rseq-x86.h>
|
||||
#elif defined(__ARMEL__)
|
||||
#include <rseq-arm.h>
|
||||
#elif defined(__PPC__)
|
||||
#include <rseq-ppc.h>
|
||||
#else
|
||||
#error unsupported target
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Register rseq for the current thread. This needs to be called once
|
||||
* by any thread which uses restartable sequences, before they start
|
||||
* using restartable sequences, to ensure restartable sequences
|
||||
* succeed. A restartable sequence executed from a non-registered
|
||||
* thread will always fail.
|
||||
*/
|
||||
int rseq_register_current_thread(void);
|
||||
|
||||
/*
|
||||
* Unregister rseq for current thread.
|
||||
*/
|
||||
int rseq_unregister_current_thread(void);
|
||||
|
||||
/*
|
||||
* Restartable sequence fallback for reading the current CPU number.
|
||||
*/
|
||||
int32_t rseq_fallback_current_cpu(void);
|
||||
|
||||
/*
|
||||
* Values returned can be either the current CPU number, -1 (rseq is
|
||||
* uninitialized), or -2 (rseq initialization has failed).
|
||||
*/
|
||||
static inline int32_t rseq_current_cpu_raw(void)
|
||||
{
|
||||
return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id);
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns a possible CPU number, which is typically the current CPU.
|
||||
* The returned CPU number can be used to prepare for an rseq critical
|
||||
* section, which will confirm whether the cpu number is indeed the
|
||||
* current one, and whether rseq is initialized.
|
||||
*
|
||||
* The CPU number returned by rseq_cpu_start should always be validated
|
||||
* by passing it to a rseq asm sequence, or by comparing it to the
|
||||
* return value of rseq_current_cpu_raw() if the rseq asm sequence
|
||||
* does not need to be invoked.
|
||||
*/
|
||||
static inline uint32_t rseq_cpu_start(void)
|
||||
{
|
||||
return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id_start);
|
||||
}
|
||||
|
||||
static inline uint32_t rseq_current_cpu(void)
|
||||
{
|
||||
int32_t cpu;
|
||||
|
||||
cpu = rseq_current_cpu_raw();
|
||||
if (rseq_unlikely(cpu < 0))
|
||||
cpu = rseq_fallback_current_cpu();
|
||||
return cpu;
|
||||
}
|
||||
|
||||
/*
|
||||
* rseq_prepare_unload() should be invoked by each thread using rseq_finish*()
|
||||
* at least once between their last rseq_finish*() and library unload of the
|
||||
* library defining the rseq critical section (struct rseq_cs). This also
|
||||
* applies to use of rseq in code generated by JIT: rseq_prepare_unload()
|
||||
* should be invoked at least once by each thread using rseq_finish*() before
|
||||
* reclaim of the memory holding the struct rseq_cs.
|
||||
*/
|
||||
static inline void rseq_prepare_unload(void)
|
||||
{
|
||||
__rseq_abi.rseq_cs = 0;
|
||||
}
|
||||
|
||||
#endif /* RSEQ_H_ */
|
||||
Loading…
Reference in New Issue
Block a user