mirrors/gcc
0
0
Fork 0
mirror of https://gcc.gnu.org/git/gcc.git synced 2024-11-27 22:03:57 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

libitm: Conversion to c++11 atomics.

Browse Source 
* local_atomic: New file.
        * libitm_i.h: Include it.
        (gtm_thread::shared_state): Use atomic template.
        * beginend.cc (GTM::gtm_clock): Use atomic template.
        (global_tid): Use atomic template if 64-bit atomics available.
        (gtm_thread::gtm_thread): Update shared_state access.
        (gtm_thread::trycommit): Likewise.
        (choose_code_path): Update global_tid access.
        * method-gl.cc (gl_mg::orec): Use atomic template.  Update all users.
        * stmlock.h (GTM::gtm_clock): Use atomic template.
        (gtm_get_clock, gtm_inc_clock): Update accesses.
        * config/linux/rwlock.cc (gtm_rwlock::read_lock): Remove
        redundant __sync_synchronize after atomic shared_state access.
        * config/posix/rwlock.cc (gtm_rwlock::read_lock): Likewise.
        (gtm_rwlock::write_lock_generic): Likewise.
        (gtm_rwlock::read_unlock): Likewise.
        * config/alpha/target.h (atomic_read_barrier): Remove.
        (atomic_write_barrier): Remove.
        * config/x86/target.h (atomic_read_barrier): Remove.
        (atomic_write_barrier): Remove.

From-SVN: r182294
This commit is contained in:
Richard Henderson 2011-12-13 11:11:25 -08:00 committed by Richard Henderson
parent c36cc670b5
commit 36cfbee133
10 changed files with 1996 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
libitm/ChangeLog
View File

@ -1,3 +1,26 @@
2011-12-13 Richard Henderson <rth@redhat.com>
* local_atomic: New file.
* libitm_i.h: Include it.
(gtm_thread::shared_state): Use atomic template.
* beginend.cc (GTM::gtm_clock): Use atomic template.
(global_tid): Use atomic template if 64-bit atomics available.
(gtm_thread::gtm_thread): Update shared_state access.
(gtm_thread::trycommit): Likewise.
(choose_code_path): Update global_tid access.
* method-gl.cc (gl_mg::orec): Use atomic template. Update all users.
* stmlock.h (GTM::gtm_clock): Use atomic template.
(gtm_get_clock, gtm_inc_clock): Update accesses.
* config/linux/rwlock.cc (gtm_rwlock::read_lock): Remove
redundant __sync_synchronize after atomic shared_state access.
* config/posix/rwlock.cc (gtm_rwlock::read_lock): Likewise.
(gtm_rwlock::write_lock_generic): Likewise.
(gtm_rwlock::read_unlock): Likewise.
* config/alpha/target.h (atomic_read_barrier): Remove.
(atomic_write_barrier): Remove.
* config/x86/target.h (atomic_read_barrier): Remove.
(atomic_write_barrier): Remove.
2011-11-30 Richard Henderson <rth@redhat.com>
* libitm_i.h (GTM_longjmp): Swap first and second arguments.

23
libitm/beginend.cc
View File

@ -37,12 +37,18 @@ gtm_thread *GTM::gtm_thread::list_of_threads = 0;
unsigned GTM::gtm_thread::number_of_threads = 0;
gtm_stmlock GTM::gtm_stmlock_array[LOCK_ARRAY_SIZE];
gtm_version GTM::gtm_clock;
atomic<gtm_version> GTM::gtm_clock;
/* ??? Move elsewhere when we figure out library initialization. */
uint64_t GTM::gtm_spin_count_var = 1000;
#ifdef HAVE_64BIT_SYNC_BUILTINS
static atomic<_ITM_transactionId_t> global_tid;
#else
static _ITM_transactionId_t global_tid;
static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
#endif
// Provides a on-thread-exit callback used to release per-thread data.
static pthread_key_t thr_release_key;
@ -114,7 +120,7 @@ GTM::gtm_thread::gtm_thread ()
// This object's memory has been set to zero by operator new, so no need
// to initialize any of the other primitive-type members that do not have
// constructors.
shared_state = ~(typeof shared_state)0;
shared_state.store(-1, memory_order_relaxed);
// Register this transaction with the list of all threads' transactions.
serial_lock.write_lock ();
@ -132,13 +138,8 @@ GTM::gtm_thread::gtm_thread ()
GTM_fatal("Setting thread release TLS key failed.");
}
#ifndef HAVE_64BIT_SYNC_BUILTINS
static pthread_mutex_t global_tid_lock = PTHREAD_MUTEX_INITIALIZER;
#endif
static inline uint32_t choose_code_path(uint32_t prop, abi_dispatch *disp)
static inline uint32_t
choose_code_path(uint32_t prop, abi_dispatch *disp)
{
if ((prop & pr_uninstrumentedCode) && disp->can_run_uninstrumented_code())
return a_runUninstrumentedCode;
@ -258,7 +259,7 @@ GTM::gtm_thread::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
else
{
#ifdef HAVE_64BIT_SYNC_BUILTINS
tx->id = __sync_add_and_fetch (&global_tid, tid_block_size);
tx->id = global_tid.fetch_add(tid_block_size, memory_order_relaxed);
tx->local_tid = tx->id + 1;
#else
pthread_mutex_lock (&global_tid_lock);
@ -480,7 +481,7 @@ GTM::gtm_thread::trycommit ()
it = it->next_thread)
{
if (it == this) continue;
while (it->shared_state < priv_time)
while (it->shared_state.load(memory_order_relaxed) < priv_time)
cpu_relax();
}
}

12
libitm/config/alpha/target.h
View File

@ -45,16 +45,4 @@ cpu_relax (void)
__asm volatile ("" : : : "memory");
}
static inline void
atomic_read_barrier (void)
{
__sync_synchronize ();
}
static inline void
atomic_write_barrier (void)
{
__asm volatile ("wmb" : : : "memory");
}
} // namespace GTM

10
libitm/config/linux/rwlock.cc
View File

@ -36,10 +36,9 @@ gtm_rwlock::read_lock (gtm_thread *tx)
for (;;)
{
// Fast path: first announce our intent to read, then check for
// conflicting intents to write. The barrier makes sure that this
// happens in exactly this order.
// conflicting intents to write. Note that direct assignment to
// an atomic object is memory_order_seq_cst.
tx->shared_state = 0;
__sync_synchronize();
if (likely(writers == 0))
return;
@ -51,8 +50,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
// We need the barrier here for the same reason that we need it in
// read_unlock().
// TODO Potentially too many wake-ups. See comments in read_unlock().
tx->shared_state = ~(typeof tx->shared_state)0;
__sync_synchronize();
tx->shared_state = -1;
if (writer_readers > 0)
{
writer_readers = 0;
@ -71,7 +69,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
// are no writers anymore after the barrier because this pending
// store could then lead to lost wake-ups at other readers.
readers = 1;
__sync_synchronize();
atomic_thread_fence(memory_order_acq_rel);
if (writers)
futex_wait(&readers, 1);
}

18
libitm/config/posix/rwlock.cc
View File

@ -53,10 +53,9 @@ void
gtm_rwlock::read_lock (gtm_thread *tx)
{
// Fast path: first announce our intent to read, then check for conflicting
// intents to write. The barrier makes sure that this happens in exactly
// this order.
// intents to write. Note that direct assignment to an atomic object
// is memory_order_seq_cst.
tx->shared_state = 0;
__sync_synchronize();
unsigned int sum = this->summary;
if (likely(!(sum & (a_writer | w_writer))))
return;
@ -69,7 +68,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
// to happen before we leave the slow path and before we wait for any
// writer).
// ??? Add a barrier to enforce early visibility of this?
tx->shared_state = ~(typeof tx->shared_state)0;
tx->shared_state.store(-1, memory_order_relaxed);
pthread_mutex_lock (&this->mutex);
@ -101,7 +100,7 @@ gtm_rwlock::read_lock (gtm_thread *tx)
}
// Otherwise we can acquire the lock for read.
tx->shared_state = 0;
tx->shared_state.store(0, memory_order_relaxed);
pthread_mutex_unlock(&this->mutex);
}
@ -153,11 +152,11 @@ gtm_rwlock::write_lock_generic (gtm_thread *tx)
// sure that we first set our write intent and check for active readers
// after that, in strictly this order (similar to the barrier in the fast
// path of read_lock()).
__sync_synchronize();
atomic_thread_fence(memory_order_acq_rel);
// If this is an upgrade, we are not a reader anymore.
if (tx != 0)
tx->shared_state = ~(typeof tx->shared_state)0;
tx->shared_state.store(-1, memory_order_relaxed);
// Count the number of active readers to be able to decrease the number of
// wake-ups and wait calls that are necessary.
@ -194,7 +193,7 @@ gtm_rwlock::write_lock_generic (gtm_thread *tx)
it = it->next_thread)
{
// Don't count ourself if this is an upgrade.
if (it->shared_state != ~(typeof it->shared_state)0)
if (it->shared_state.load(memory_order_relaxed) != -1)
readers++;
}
@ -236,8 +235,7 @@ gtm_rwlock::write_upgrade (gtm_thread *tx)
void
gtm_rwlock::read_unlock (gtm_thread *tx)
{
tx->shared_state = ~(typeof tx->shared_state)0;
__sync_synchronize();
tx->shared_state = -1;
unsigned int sum = this->summary;
if (likely(!(sum & (a_writer | w_writer))))
return;

14
libitm/config/x86/target.h
View File

@ -66,20 +66,6 @@ cpu_relax (void)
__asm volatile ("rep; nop" : : : "memory");
}
static inline void
atomic_read_barrier (void)
{
/* x86 is a strong memory ordering machine. */
__asm volatile ("" : : : "memory");
}
static inline void
atomic_write_barrier (void)
{
/* x86 is a strong memory ordering machine. */
__asm volatile ("" : : : "memory");
}
} // namespace GTM
// We'll be using some of the cpu builtins, and their associated types.

3
libitm/libitm_i.h
View File

@ -37,6 +37,7 @@
#include <string.h>
#include <unwind.h>
#include "local_type_traits"
#include "local_atomic"
#include "common.h"
@ -206,7 +207,7 @@ struct gtm_thread
// If this transaction is inactive, shared_state is ~0. Otherwise, this is
// an active or serial transaction.
gtm_word shared_state;
atomic<gtm_word> shared_state;
// The lock that provides access to serial mode. Non-serialized
// transactions acquire read locks; a serialized transaction aquires

1903
libitm/local_atomic Normal file
View File

File diff suppressed because it is too large Load Diff

75
libitm/method-gl.cc
View File

@ -41,10 +41,11 @@ struct gl_mg : public method_group
static gtm_word clear_locked(gtm_word l) { return l & ~LOCK_BIT; }
// The global ownership record.
gtm_word orec;
atomic<gtm_word> orec;
virtual void init()
{
orec = 0;
orec.store(0, memory_order_relaxed);
}
virtual void fini() { }
};
@ -84,28 +85,25 @@ protected:
static void pre_write(const void *addr, size_t len)
{
gtm_thread *tx = gtm_thr();
if (unlikely(!gl_mg::is_locked(tx->shared_state)))
gtm_word v = tx->shared_state.load(memory_order_acquire);
if (unlikely(!gl_mg::is_locked(v)))
{
// Check for and handle version number overflow.
if (unlikely(tx->shared_state >= gl_mg::VERSION_MAX))
if (unlikely(v >= gl_mg::VERSION_MAX))
tx->restart(RESTART_INIT_METHOD_GROUP);
// CAS global orec from our snapshot time to the locked state.
// This validates that we have a consistent snapshot, which is also
// for making privatization safety work (see the class' comments).
gtm_word now = o_gl_mg.orec;
if (now != tx->shared_state)
gtm_word now = o_gl_mg.orec.load(memory_order_relaxed);
if (now != v)
tx->restart(RESTART_VALIDATE_WRITE);
if (__sync_val_compare_and_swap(&o_gl_mg.orec, now,
gl_mg::set_locked(now)) != now)
if (!o_gl_mg.orec.compare_exchange_strong (now, gl_mg::set_locked(now),
memory_order_acquire))
tx->restart(RESTART_LOCKED_WRITE);
// Set shared_state to new value. The CAS is a full barrier, so the
// acquisition of the global orec is visible before this store here,
// and the store will not be visible before earlier data loads, which
// is required to correctly ensure privatization safety (see
// begin_and_restart() and release_orec() for further comments).
tx->shared_state = gl_mg::set_locked(now);
// Set shared_state to new value.
tx->shared_state.store(gl_mg::set_locked(now), memory_order_release);
}
// TODO Ensure that this gets inlined: Use internal log interface and LTO.
@ -115,11 +113,12 @@ protected:
static void validate()
{
// Check that snapshot is consistent. The barrier ensures that this
// happens after previous data loads.
atomic_read_barrier();
// happens after previous data loads. Recall that load cannot itself
// have memory_order_release.
gtm_thread *tx = gtm_thr();
gtm_word l = o_gl_mg.orec;
if (l != tx->shared_state)
atomic_thread_fence(memory_order_release);
gtm_word l = o_gl_mg.orec.load(memory_order_relaxed);
if (l != tx->shared_state.load(memory_order_relaxed))
tx->restart(RESTART_VALIDATE_READ);
}
@ -180,17 +179,18 @@ public:
// Spin until global orec is not locked.
// TODO This is not necessary if there are no pure loads (check txn props).
gtm_word v;
unsigned i = 0;
while (gl_mg::is_locked(v = o_gl_mg.orec))
gtm_word v;
while (1)
{
v = o_gl_mg.orec.load(memory_order_acquire);
if (!gl_mg::is_locked(v))
break;
// TODO need method-specific max spin count
if (++i > gtm_spin_count_var) return RESTART_VALIDATE_READ;
if (++i > gtm_spin_count_var)
return RESTART_VALIDATE_READ;
cpu_relax();
}
// This barrier ensures that we have read the global orec before later
// data loads.
atomic_read_barrier();
// Everything is okay, we have a snapshot time.
// We don't need to enforce any ordering for the following store. There
@ -202,14 +202,14 @@ public:
// marking the transaction as active, and restarts enforce immediate
// visibility of a smaller or equal value with a barrier (see
// release_orec()).
tx->shared_state = v;
tx->shared_state.store(v, memory_order_relaxed);
return NO_RESTART;
}
virtual bool trycommit(gtm_word& priv_time)
{
gtm_thread* tx = gtm_thr();
gtm_word v = tx->shared_state;
gtm_word v = tx->shared_state.load(memory_order_acquire);
// Special case: If shared_state is ~0, then we have acquired the
// serial lock (tx->state is not updated yet). In this case, the previous
@ -218,7 +218,7 @@ public:
// anymore. In particular, if it is locked, then we are an update
// transaction, which is all we care about for commit.
if (v == ~(typeof v)0)
v = o_gl_mg.orec;
v = o_gl_mg.orec.load(memory_order_relaxed);
// Release the orec but do not reset shared_state, which will be modified
// by the serial lock right after our commit anyway. Also, resetting
@ -227,10 +227,8 @@ public:
if (gl_mg::is_locked(v))
{
// Release the global orec, increasing its version number / timestamp.
// TODO replace with C++0x-style atomics (a release in this case)
atomic_write_barrier();
v = gl_mg::clear_locked(v) + 1;
o_gl_mg.orec = v;
o_gl_mg.orec.store(v, memory_order_release);
// Need to ensure privatization safety. Every other transaction must
// have a snapshot time that is at least as high as our commit time
@ -247,15 +245,16 @@ public:
return;
gtm_thread *tx = gtm_thr();
gtm_word v = tx->shared_state;
gtm_word v = tx->shared_state.load(memory_order_acquire);
// Special case: If shared_state is ~0, then we have acquired the
// serial lock (tx->state is not updated yet). In this case, the previous
// value isn't available anymore, so grab it from the global lock, which
// must have a meaningful value because no other transactions are active
// anymore. In particular, if it is locked, then we are an update
// transaction, which is all we care about for rollback.
if (v == ~(typeof v)0)
v = o_gl_mg.orec;
bool is_serial = v == ~(typeof v)0;
if (is_serial)
v = o_gl_mg.orec.load(memory_order_relaxed);
// Release lock and increment version number to prevent dirty reads.
// Also reset shared state here, so that begin_or_restart() can expect a
@ -263,16 +262,14 @@ public:
if (gl_mg::is_locked(v))
{
// Release the global orec, increasing its version number / timestamp.
// TODO replace with C++0x-style atomics (a release in this case)
atomic_write_barrier();
v = gl_mg::clear_locked(v) + 1;
o_gl_mg.orec = v;
o_gl_mg.orec.store(v, memory_order_release);
// Also reset the timestamp published via shared_state.
// Special case: Only do this if we are not a serial transaction
// because otherwise, we would interfere with the serial lock.
if (tx->shared_state != ~(typeof tx->shared_state)0)
tx->shared_state = v;
if (!is_serial)
tx->shared_state.store(v, memory_order_relaxed);
// We need a store-load barrier after this store to prevent it
// from becoming visible after later data loads because the
@ -280,7 +277,7 @@ public:
// snapshot time (the lock bit had been set), which could break
// privatization safety. We do not need a barrier before this
// store (see pre_write() for an explanation).
__sync_synchronize();
atomic_thread_fence(memory_order_acq_rel);
}
}

17
libitm/stmlock.h
View File

@ -92,24 +92,23 @@ gtm_get_stmlock (const gtm_cacheline *addr)
}
/* The current global version number. */
extern gtm_version gtm_clock;
extern atomic<gtm_version> gtm_clock;
static inline gtm_version
gtm_get_clock (void)
{
gtm_version r;
__sync_synchronize ();
r = gtm_clock;
atomic_read_barrier ();
return r;
atomic_thread_fence(memory_order_release);
return gtm_clock.load(memory_order_acquire);
}
static inline gtm_version
gtm_inc_clock (void)
{
gtm_version r = __sync_add_and_fetch (&gtm_clock, 1);
/* ??? Here we have a choice, the pre-inc operator mapping to
__atomic_add_fetch with memory_order_seq_cst, or fetch_add
with memory_order_acq_rel plus another separate increment.
We really ought to recognize and optimize fetch_op(x) op x... */
gtm_version r = ++gtm_clock;
/* ??? Ought to handle wraparound for 32-bit. */
if (sizeof(r) < 8 && r > GTM_VERSION_MAX)