mirrors/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2024-11-27 03:33:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

nptl: Move stack list variables into _rtld_global

Browse Source 
Now __thread_gscope_wait (the function behind THREAD_GSCOPE_WAIT,
formerly __wait_lookup_done) can be implemented directly in ld.so,
eliminating the unprotected GL (dl_wait_lookup_done) function
pointer.

Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>
This commit is contained in:
Florian Weimer 2020-11-16 19:33:30 +01:00
parent aac0f62c47
commit 1daccf403b
37 changed files with 243 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
csu/libc-tls.c
View File

@ -24,6 +24,7 @@
#include <stdio.h>
#include <sys/param.h>
#include <array_length.h>
#include <list.h>
#ifdef SHARED
#error makefile bug, this file is for static only
@ -193,6 +194,12 @@ __libc_setup_tls (void)
if (__builtin_expect (lossage != NULL, 0))
_startup_fatal (lossage);
#if THREAD_GSCOPE_IN_TCB
INIT_LIST_HEAD (&_dl_stack_used);
INIT_LIST_HEAD (&_dl_stack_user);
list_add (&THREAD_SELF->list, &_dl_stack_user);
#endif
/* Update the executable's link map with enough information to make
the TLS routines happy. */
main_map->l_tls_align = align;

6
elf/Makefile
View File

@ -34,7 +34,8 @@ dl-routines = $(addprefix dl-,load lookup object reloc deps \
version profile tls origin scope \
execstack open close trampoline \
exception sort-maps lookup-direct \
call-libc-early-init write)
call-libc-early-init write \
thread_gscope_wait)
ifeq (yes,$(use-ldconfig))
dl-routines += dl-cache
endif
@ -54,7 +55,8 @@ endif
all-dl-routines = $(dl-routines) $(sysdep-dl-routines)
# But they are absent from the shared libc, because that code is in ld.so.
elide-routines.os = $(all-dl-routines) dl-support enbl-secure dl-origin \
dl-sysdep dl-exception dl-reloc-static-pie
dl-sysdep dl-exception dl-reloc-static-pie \
thread_gscope_wait
# ld.so uses those routines, plus some special stuff for being the program
# interpreter and operating independent of libc.

9
elf/dl-support.c
View File

@ -183,10 +183,11 @@ ElfW(Word) _dl_stack_flags = DEFAULT_STACK_PERMS;
int (*_dl_make_stack_executable_hook) (void **) = _dl_make_stack_executable;
/* Function in libpthread to wait for termination of lookups. */
void (*_dl_wait_lookup_done) (void);
#if !THREAD_GSCOPE_IN_TCB
#if THREAD_GSCOPE_IN_TCB
list_t _dl_stack_used;
list_t _dl_stack_user;
int _dl_stack_cache_lock;
#else
int _dl_thread_gscope_count;
#endif
struct dl_scope_free_list *_dl_scope_free_list;

2
elf/dl-thread_gscope_wait.c Normal file
View File

@ -0,0 +1,2 @@
/* By default, the dynamic linker does not use an out-of-line
__thread_gscope_wait function. */

12
elf/rtld.c
View File

@ -48,6 +48,7 @@
#include <array_length.h>
#include <libc-early-init.h>
#include <dl-main.h>
#include <list.h>
#include <assert.h>
@ -799,6 +800,9 @@ cannot allocate TLS data structures for initial thread\n");
const char *lossage = TLS_INIT_TP (tcbp);
if (__glibc_unlikely (lossage != NULL))
_dl_fatal_printf ("cannot set up thread-local storage: %s\n", lossage);
#if THREAD_GSCOPE_IN_TCB
list_add (&THREAD_SELF->list, &GL (dl_stack_user));
#endif
tls_init_tp_called = true;
return tcbp;
@ -1139,6 +1143,11 @@ dl_main (const ElfW(Phdr) *phdr,
GL(dl_rtld_unlock_recursive) = rtld_lock_default_unlock_recursive;
#endif
#if THREAD_GSCOPE_IN_TCB
INIT_LIST_HEAD (&GL (dl_stack_used));
INIT_LIST_HEAD (&GL (dl_stack_user));
#endif
/* The explicit initialization here is cheaper than processing the reloc
in the _rtld_local definition's initializer. */
GL(dl_make_stack_executable_hook) = &_dl_make_stack_executable;
@ -2383,6 +2392,9 @@ dl_main (const ElfW(Phdr) *phdr,
if (__glibc_unlikely (lossage != NULL))
_dl_fatal_printf ("cannot set up thread-local storage: %s\n",
lossage);
#if THREAD_GSCOPE_IN_TCB
list_add (&THREAD_SELF->list, &GL (dl_stack_user));
#endif
}
/* Make sure no new search directories have been added. */

147
nptl/allocatestack.c
View File

@ -106,26 +106,14 @@
static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */
static size_t stack_cache_actsize;
/* Mutex protecting this variable. */
static int stack_cache_lock = LLL_LOCK_INITIALIZER;
/* List of queued stack frames. */
static LIST_HEAD (stack_cache);
/* List of the stacks in use. */
static LIST_HEAD (stack_used);
/* We need to record what list operations we are going to do so that,
in case of an asynchronous interruption due to a fork() call, we
can correct for the work. */
static uintptr_t in_flight_stack;
/* List of the threads with user provided stacks in use. No need to
initialize this, since it's done in __pthread_initialize_minimal. */
list_t __stack_user __attribute__ ((nocommon));
hidden_data_def (__stack_user)
/* Check whether the stack is still used or not. */
#define FREE_P(descr) ((descr)->tid <= 0)
@ -173,7 +161,7 @@ get_cached_stack (size_t *sizep, void **memp)
struct pthread *result = NULL;
list_t *entry;
lll_lock (stack_cache_lock, LLL_PRIVATE);
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
/* Search the cache for a matching entry. We search for the
smallest stack which has at least the required size. Note that
@ -206,7 +194,7 @@ get_cached_stack (size_t *sizep, void **memp)
|| __builtin_expect (result->stackblock_size > 4 * size, 0))
{
/* Release the lock. */
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
return NULL;
}
@ -218,13 +206,13 @@ get_cached_stack (size_t *sizep, void **memp)
stack_list_del (&result->list);
/* And add to the list of stacks in use. */
stack_list_add (&result->list, &stack_used);
stack_list_add (&result->list, &GL (dl_stack_used));
/* And decrease the cache size. */
stack_cache_actsize -= result->stackblock_size;
/* Release the lock early. */
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
/* Report size and location of the stack to the caller. */
*sizep = result->stackblock_size;
@ -510,12 +498,12 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
/* Prepare to modify global data. */
lll_lock (stack_cache_lock, LLL_PRIVATE);
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
/* And add to the list of stacks in use. */
list_add (&pd->list, &__stack_user);
list_add (&pd->list, &GL (dl_stack_user));
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
}
else
{
@ -644,12 +632,12 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
/* Prepare to modify global data. */
lll_lock (stack_cache_lock, LLL_PRIVATE);
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
/* And add to the list of stacks in use. */
stack_list_add (&pd->list, &stack_used);
stack_list_add (&pd->list, &GL (dl_stack_used));
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
/* There might have been a race. Another thread might have
@ -690,12 +678,12 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
if (__mprotect (guard, guardsize, PROT_NONE) != 0)
{
mprot_error:
lll_lock (stack_cache_lock, LLL_PRIVATE);
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
/* Remove the thread from the list. */
stack_list_del (&pd->list);
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
/* Get rid of the TLS block we allocated. */
_dl_deallocate_tls (TLS_TPADJ (pd), false);
@ -799,7 +787,7 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
void
__deallocate_stack (struct pthread *pd)
{
lll_lock (stack_cache_lock, LLL_PRIVATE);
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
/* Remove the thread from the list of threads with user defined
stacks. */
@ -815,7 +803,7 @@ __deallocate_stack (struct pthread *pd)
/* Free the memory associated with the ELF TLS. */
_dl_deallocate_tls (TLS_TPADJ (pd), false);
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
}
@ -831,10 +819,10 @@ __make_stacks_executable (void **stack_endp)
const size_t pagemask = ~(__getpagesize () - 1);
#endif
lll_lock (stack_cache_lock, LLL_PRIVATE);
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
list_t *runp;
list_for_each (runp, &stack_used)
list_for_each (runp, &GL (dl_stack_used))
{
err = change_stack_perm (list_entry (runp, struct pthread, list)
#ifdef NEED_SEPARATE_REGISTER_STACK
@ -860,7 +848,7 @@ __make_stacks_executable (void **stack_endp)
break;
}
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
return err;
}
@ -891,8 +879,8 @@ __reclaim_stacks (void)
pointers at the head of the list are inconsistent. */
list_t *l = NULL;
if (stack_used.next->prev != &stack_used)
l = &stack_used;
if (GL (dl_stack_used).next->prev != &GL (dl_stack_used))
l = &GL (dl_stack_used);
else if (stack_cache.next->prev != &stack_cache)
l = &stack_cache;
@ -914,7 +902,7 @@ __reclaim_stacks (void)
/* Mark all stacks except the still running one as free. */
list_t *runp;
list_for_each (runp, &stack_used)
list_for_each (runp, &GL (dl_stack_used))
{
struct pthread *curp = list_entry (runp, struct pthread, list);
if (curp != self)
@ -948,7 +936,7 @@ __reclaim_stacks (void)
}
/* Add the stack of all running threads to the cache. */
list_splice (&stack_used, &stack_cache);
list_splice (&GL (dl_stack_used), &stack_cache);
/* Remove the entry for the current thread to from the cache list
and add it to the list of running threads. Which of the two
@ -956,13 +944,13 @@ __reclaim_stacks (void)
stack_list_del (&self->list);
/* Re-initialize the lists for all the threads. */
INIT_LIST_HEAD (&stack_used);
INIT_LIST_HEAD (&__stack_user);
INIT_LIST_HEAD (&GL (dl_stack_used));
INIT_LIST_HEAD (&GL (dl_stack_user));
if (__glibc_unlikely (THREAD_GETMEM (self, user_stack)))
list_add (&self->list, &__stack_user);
list_add (&self->list, &GL (dl_stack_user));
else
list_add (&self->list, &stack_used);
list_add (&self->list, &GL (dl_stack_used));
/* There is one thread running. */
__nptl_nthreads = 1;
@ -970,7 +958,7 @@ __reclaim_stacks (void)
in_flight_stack = 0;
/* Initialize locks. */
stack_cache_lock = LLL_LOCK_INITIALIZER;
GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER;
__default_pthread_attr_lock = LLL_LOCK_INITIALIZER;
}
@ -1083,7 +1071,7 @@ __nptl_setxid (struct xid_command *cmdp)
{
int signalled;
int result;
lll_lock (stack_cache_lock, LLL_PRIVATE);
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
__xidcmd = cmdp;
cmdp->cntr = 0;
@ -1093,7 +1081,7 @@ __nptl_setxid (struct xid_command *cmdp)
/* Iterate over the list with system-allocated threads first. */
list_t *runp;
list_for_each (runp, &stack_used)
list_for_each (runp, &GL (dl_stack_used))
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self)
@ -1103,7 +1091,7 @@ __nptl_setxid (struct xid_command *cmdp)
}
/* Now the list with threads using user-allocated stacks. */
list_for_each (runp, &__stack_user)
list_for_each (runp, &GL (dl_stack_user))
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self)
@ -1119,7 +1107,7 @@ __nptl_setxid (struct xid_command *cmdp)
{
signalled = 0;
list_for_each (runp, &stack_used)
list_for_each (runp, &GL (dl_stack_used))
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self)
@ -1128,7 +1116,7 @@ __nptl_setxid (struct xid_command *cmdp)
signalled += setxid_signal_thread (cmdp, t);
}
list_for_each (runp, &__stack_user)
list_for_each (runp, &GL (dl_stack_user))
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self)
@ -1149,7 +1137,7 @@ __nptl_setxid (struct xid_command *cmdp)
/* Clean up flags, so that no thread blocks during exit waiting
for a signal which will never come. */
list_for_each (runp, &stack_used)
list_for_each (runp, &GL (dl_stack_used))
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self)
@ -1158,7 +1146,7 @@ __nptl_setxid (struct xid_command *cmdp)
setxid_unmark_thread (cmdp, t);
}
list_for_each (runp, &__stack_user)
list_for_each (runp, &GL (dl_stack_user))
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self)
@ -1180,7 +1168,7 @@ __nptl_setxid (struct xid_command *cmdp)
}
__nptl_setxid_error (cmdp, error);
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
return result;
}
@ -1204,75 +1192,16 @@ void
attribute_hidden
__pthread_init_static_tls (struct link_map *map)
{
lll_lock (stack_cache_lock, LLL_PRIVATE);
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
/* Iterate over the list with system-allocated threads first. */
list_t *runp;
list_for_each (runp, &stack_used)
list_for_each (runp, &GL (dl_stack_used))
init_one_static_tls (list_entry (runp, struct pthread, list), map);
/* Now the list with threads using user-allocated stacks. */
list_for_each (runp, &__stack_user)
list_for_each (runp, &GL (dl_stack_user))
init_one_static_tls (list_entry (runp, struct pthread, list), map);
lll_unlock (stack_cache_lock, LLL_PRIVATE);
}
void
attribute_hidden
__wait_lookup_done (void)
{
lll_lock (stack_cache_lock, LLL_PRIVATE);
struct pthread *self = THREAD_SELF;
/* Iterate over the list with system-allocated threads first. */
list_t *runp;
list_for_each (runp, &stack_used)
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
continue;
int *const gscope_flagp = &t->header.gscope_flag;
/* We have to wait until this thread is done with the global
scope. First tell the thread that we are waiting and
possibly have to be woken. */
if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
THREAD_GSCOPE_FLAG_WAIT,
THREAD_GSCOPE_FLAG_USED))
continue;
do
futex_wait_simple ((unsigned int *) gscope_flagp,
THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
}
/* Now the list with threads using user-allocated stacks. */
list_for_each (runp, &__stack_user)
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
continue;
int *const gscope_flagp = &t->header.gscope_flag;
/* We have to wait until this thread is done with the global
scope. First tell the thread that we are waiting and
possibly have to be woken. */
if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
THREAD_GSCOPE_FLAG_WAIT,
THREAD_GSCOPE_FLAG_USED))
continue;
do
futex_wait_simple ((unsigned int *) gscope_flagp,
THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
}
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
}

3
nptl/descr.h
View File

@ -162,7 +162,8 @@ struct pthread
void *__padding[24];
};
/* This descriptor's link on the `stack_used' or `__stack_user' list. */
/* This descriptor's link on the GL (dl_stack_used) or
GL (dl_stack_user) list. */
list_t list;
/* Thread ID - which is also a 'is this thread descriptor (and

11
nptl/nptl-init.c
View File

@ -251,12 +251,9 @@ __pthread_initialize_minimal_internal (void)
purposes this is good enough. */
THREAD_SETMEM (pd, stackblock_size, (size_t) __libc_stack_end);
/* Initialize the list of all running threads with the main thread. */
INIT_LIST_HEAD (&__stack_user);
list_add (&pd->list, &__stack_user);
/* Before initializing __stack_user, the debugger could not find us and
had to set __nptl_initial_report_events. Propagate its setting. */
/* Before initializing GL (dl_stack_user), the debugger could not
find us and had to set __nptl_initial_report_events. Propagate
its setting. */
THREAD_SETMEM (pd, report_events, __nptl_initial_report_events);
struct sigaction sa;
@ -336,8 +333,6 @@ __pthread_initialize_minimal_internal (void)
GL(dl_init_static_tls) = &__pthread_init_static_tls;
GL(dl_wait_lookup_done) = &__wait_lookup_done;
/* Register the fork generation counter with the libc. */
#ifndef TLS_MULTIPLE_THREADS_IN_TCB
__libc_multiple_threads_ptr =

4
nptl/pthreadP.h
View File

@ -208,10 +208,6 @@ extern void __default_pthread_attr_freeres (void) attribute_hidden;
extern size_t __static_tls_size attribute_hidden;
extern size_t __static_tls_align_m1 attribute_hidden;
/* Thread descriptor handling. */
extern list_t __stack_user;
hidden_proto (__stack_user)
/* Attribute handling. */
extern struct pthread_attr *__attr_list attribute_hidden;
extern int __attr_list_lock attribute_hidden;

8
nptl/pthread_create.c
View File

@ -213,9 +213,9 @@ __find_in_stack_list (struct pthread *pd)
list_t *entry;
struct pthread *result = NULL;
lll_lock (stack_cache_lock, LLL_PRIVATE);
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
list_for_each (entry, &stack_used)
list_for_each (entry, &GL (dl_stack_used))
{
struct pthread *curp;
@ -228,7 +228,7 @@ __find_in_stack_list (struct pthread *pd)
}
if (result == NULL)
list_for_each (entry, &__stack_user)
list_for_each (entry, &GL (dl_stack_user))
{
struct pthread *curp;
@ -240,7 +240,7 @@ __find_in_stack_list (struct pthread *pd)
}
}
lll_unlock (stack_cache_lock, LLL_PRIVATE);
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
return result;
}

4
nptl_db/structs.def
View File

@ -69,8 +69,6 @@ DB_STRUCT (td_eventbuf_t)
DB_STRUCT_FIELD (td_eventbuf_t, eventnum)
DB_STRUCT_FIELD (td_eventbuf_t, eventdata)
DB_SYMBOL (stack_used)
DB_SYMBOL (__stack_user)
DB_SYMBOL (nptl_version)
DB_FUNCTION (__nptl_create_event)
DB_FUNCTION (__nptl_death_event)
@ -106,6 +104,8 @@ DB_STRUCT (rtld_global)
DB_RTLD_VARIABLE (_rtld_global)
#endif
DB_RTLD_GLOBAL_FIELD (dl_tls_dtv_slotinfo_list)
DB_RTLD_GLOBAL_FIELD (dl_stack_user)
DB_RTLD_GLOBAL_FIELD (dl_stack_used)
DB_STRUCT (dtv_slotinfo_list)
DB_STRUCT_FIELD (dtv_slotinfo_list, len)

14
nptl_db/td_init.c
View File

@ -29,3 +29,17 @@ td_init (void)
LOG ("td_init");
return TD_OK;
}
bool
__td_ta_rtld_global (td_thragent_t *ta)
{
if (ta->ta_addr__rtld_global == 0
&& td_mod_lookup (ta->ph, LD_SO, SYM__rtld_global,
&ta->ta_addr__rtld_global) != PS_OK)
{
ta->ta_addr__rtld_global = (void*)-1;
return false;
}
else
return ta->ta_addr__rtld_global != (void*)-1;
}

2
nptl_db/td_ta_map_lwp2thr.c
View File

@ -187,7 +187,7 @@ td_ta_map_lwp2thr (const td_thragent_t *ta_arg,
fake a special descriptor for the initial thread. */
psaddr_t list;
td_err_e err = DB_GET_SYMBOL (list, ta, __stack_user);
td_err_e err = __td_ta_stack_user (ta, &list);
if (err != TD_OK)
return err;

4
nptl_db/td_ta_thr_iter.c
View File

@ -133,14 +133,14 @@ td_ta_thr_iter (const td_thragent_t *ta_arg, td_thr_iter_f *callback,
have to iterate over both lists separately. We start with the
list of threads with user-defined stacks. */
err = DB_GET_SYMBOL (list, ta, __stack_user);
err = __td_ta_stack_user (ta, &list);
if (err == TD_OK)
err = iterate_thread_list (ta, callback, cbdata_p, state, ti_pri,
list, true);
/* And the threads with stacks allocated by the implementation. */
if (err == TD_OK)
err = DB_GET_SYMBOL (list, ta, stack_used);
err = __td_ta_stack_used (ta, &list);
if (err == TD_OK)
err = iterate_thread_list (ta, callback, cbdata_p, state, ti_pri,
list, false);

7
nptl_db/td_thr_tlsbase.c
View File

@ -28,12 +28,7 @@ dtv_slotinfo_list (td_thragent_t *ta,
td_err_e err;
psaddr_t head;
if (ta->ta_addr__rtld_global == 0
&& td_mod_lookup (ta->ph, LD_SO, SYM__rtld_global,
&ta->ta_addr__rtld_global) != PS_OK)
ta->ta_addr__rtld_global = (void*)-1;
if (ta->ta_addr__rtld_global != (void*)-1)
if (__td_ta_rtld_global (ta))
{
err = DB_GET_FIELD (head, ta, ta->ta_addr__rtld_global,
rtld_global, _dl_tls_dtv_slotinfo_list, 0);

39
nptl_db/td_thr_validate.c
View File

@ -20,6 +20,41 @@
#include "thread_dbP.h"
#include <stdbool.h>
td_err_e
__td_ta_stack_user (td_thragent_t *ta, psaddr_t *plist)
{
if (__td_ta_rtld_global (ta))
return DB_GET_FIELD_ADDRESS (*plist, ta, ta->ta_addr__rtld_global,
rtld_global, _dl_stack_user, 0);
else
{
if (ta->ta_addr__dl_stack_user == 0
&& td_mod_lookup (ta->ph, NULL, SYM__dl_stack_user,
&ta->ta_addr__dl_stack_user) != PS_OK)
return TD_ERR;
*plist = ta->ta_addr__dl_stack_user;
return TD_OK;
}
}
td_err_e
__td_ta_stack_used (td_thragent_t *ta, psaddr_t *plist)
{
if (__td_ta_rtld_global (ta))
return DB_GET_FIELD_ADDRESS (*plist, ta, ta->ta_addr__rtld_global,
rtld_global, _dl_stack_used, 0);
else
{
if (ta->ta_addr__dl_stack_used == 0
&& td_mod_lookup (ta->ph, NULL, SYM__dl_stack_used,
&ta->ta_addr__dl_stack_used) != PS_OK)
return TD_ERR;
*plist = ta->ta_addr__dl_stack_used;
return TD_OK;
}
}
static td_err_e
check_thread_list (const td_thrhandle_t *th, psaddr_t head, bool *uninit)
{
@ -62,7 +97,7 @@ td_thr_validate (const td_thrhandle_t *th)
/* First check the list with threads using user allocated stacks. */
bool uninit = false;
err = DB_GET_SYMBOL (list, th->th_ta_p, __stack_user);
err = __td_ta_stack_user (th->th_ta_p, &list);
if (err == TD_OK)
err = check_thread_list (th, list, &uninit);
@ -70,7 +105,7 @@ td_thr_validate (const td_thrhandle_t *th)
using implementation allocated stacks. */
if (err == TD_NOTHR)
{
err = DB_GET_SYMBOL (list, th->th_ta_p, stack_used);
err = __td_ta_stack_used (th->th_ta_p, &list);
if (err == TD_OK)
err = check_thread_list (th, list, &uninit);

13
nptl_db/thread_dbP.h
View File

@ -269,4 +269,17 @@ extern td_err_e _td_check_sizeof (td_thragent_t *ta, uint32_t *sizep,
extern td_err_e __td_ta_lookup_th_unique (const td_thragent_t *ta,
lwpid_t lwpid, td_thrhandle_t *th);
/* Try to initialize TA->ta_addr__rtld_global. Return true on
success, false on failure (which may be cached). */
bool __td_ta_rtld_global (td_thragent_t *ta) attribute_hidden;
/* Obtain the address of the list_t fields _dl_stack_user and
_dl_stack_used in _rtld_global, or fall back to the global
variables of the same name (to support statically linked
programs). */
td_err_e __td_ta_stack_user (td_thragent_t *ta, psaddr_t *plist)
attribute_hidden;
td_err_e __td_ta_stack_used (td_thragent_t *ta, psaddr_t *plist)
attribute_hidden;
#endif /* thread_dbP.h */

2
sysdeps/aarch64/nptl/tls.h
View File

@ -129,8 +129,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
# define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
# endif /* __ASSEMBLER__ */

2
sysdeps/alpha/nptl/tls.h
View File

@ -123,8 +123,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#else /* __ASSEMBLER__ */
# include <tcb-offsets.h>

2
sysdeps/arc/nptl/tls.h
View File

@ -131,8 +131,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* !__ASSEMBLER__ */

2
sysdeps/arm/nptl/tls.h
View File

@ -120,8 +120,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/csky/nptl/tls.h
View File

@ -147,8 +147,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
# define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

19
sysdeps/generic/ldsodefs.h
View File

@ -38,6 +38,7 @@
#include <libc-lock.h>
#include <hp-timing.h>
#include <tls.h>
#include <list_t.h>
__BEGIN_DECLS
@ -461,15 +462,22 @@ struct rtld_global
EXTERN void (*_dl_init_static_tls) (struct link_map *);
EXTERN void (*_dl_wait_lookup_done) (void);
/* Scopes to free after next THREAD_GSCOPE_WAIT (). */
EXTERN struct dl_scope_free_list
{
size_t count;
void *list[50];
} *_dl_scope_free_list;
#if !THREAD_GSCOPE_IN_TCB
#if THREAD_GSCOPE_IN_TCB
/* List of active thread stacks, with memory managed by glibc. */
EXTERN list_t _dl_stack_used;
/* List of thread stacks that were allocated by the application. */
EXTERN list_t _dl_stack_user;
/* Mutex protecting the stack lists. */
EXTERN int _dl_stack_cache_lock;
#else
EXTERN int _dl_thread_gscope_count;
#endif
#ifdef SHARED
@ -1252,6 +1260,11 @@ link_map_audit_state (struct link_map *l, size_t index)
}
#endif /* SHARED */
#if THREAD_GSCOPE_IN_TCB
void __thread_gscope_wait (void) attribute_hidden;
# define THREAD_GSCOPE_WAIT() __thread_gscope_wait ()
#endif
__END_DECLS
#endif /* ldsodefs.h */

2
sysdeps/hppa/nptl/tls.h
View File

@ -154,8 +154,6 @@ static inline void __set_cr27(struct pthread *cr27)
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* !__ASSEMBLER__ */

2
sysdeps/i386/nptl/tls.h
View File

@ -387,8 +387,6 @@ tls_fill_user_desc (union user_desc_init *desc,
while (0)
#define THREAD_GSCOPE_SET_FLAG() \
THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/ia64/nptl/tls.h
View File

@ -175,8 +175,6 @@ register struct pthread *__thread_self __asm__("r13");
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/m68k/nptl/tls.h
View File

@ -153,8 +153,6 @@ extern void * __m68k_read_tp (void);
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/microblaze/nptl/tls.h
View File

@ -136,8 +136,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
# define THREAD_GSCOPE_WAIT() \
GL (dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/mips/nptl/tls.h
View File

@ -178,8 +178,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/nios2/nptl/tls.h
View File

@ -157,8 +157,6 @@ register struct pthread *__thread_self __asm__("r23");
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

80
sysdeps/nptl/dl-thread_gscope_wait.c Normal file
View File

@ -0,0 +1,80 @@
/* Out-of-line notification function for the GSCOPE locking mechanism.
Copyright (C) 2007-2020 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
#include <nptl/descr.h>
#include <futex-internal.h>
#include <ldsodefs.h>
#include <list.h>
#include <lowlevellock.h>
void
__thread_gscope_wait (void)
{
lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
struct pthread *self = THREAD_SELF;
/* Iterate over the list with system-allocated threads first. */
list_t *runp;
list_for_each (runp, &GL (dl_stack_used))
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
continue;
int *const gscope_flagp = &t->header.gscope_flag;
/* We have to wait until this thread is done with the global
scope. First tell the thread that we are waiting and
possibly have to be woken. */
if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
THREAD_GSCOPE_FLAG_WAIT,
THREAD_GSCOPE_FLAG_USED))
continue;
do
futex_wait_simple ((unsigned int *) gscope_flagp,
THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
}
/* Now the list with threads using user-allocated stacks. */
list_for_each (runp, &GL (dl_stack_user))
{
struct pthread *t = list_entry (runp, struct pthread, list);
if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
continue;
int *const gscope_flagp = &t->header.gscope_flag;
/* We have to wait until this thread is done with the global
scope. First tell the thread that we are waiting and
possibly have to be woken. */
if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
THREAD_GSCOPE_FLAG_WAIT,
THREAD_GSCOPE_FLAG_USED))
continue;
do
futex_wait_simple ((unsigned int *) gscope_flagp,
THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
}
lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
}

2
sysdeps/powerpc/nptl/tls.h
View File

@ -245,8 +245,6 @@ register void *__thread_register __asm__ ("r13");
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/riscv/nptl/tls.h
View File

@ -139,8 +139,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
# define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/s390/nptl/tls.h
View File

@ -185,8 +185,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/sh/nptl/tls.h
View File

@ -161,8 +161,6 @@ typedef struct
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */

2
sysdeps/sparc/nptl/tls.h
View File

@ -158,8 +158,6 @@ register struct pthread *__thread_self __asm__("%g7");
atomic_write_barrier (); \
} \
while (0)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* !ASSEMBLER */

2
sysdeps/x86_64/nptl/tls.h
View File

@ -332,8 +332,6 @@ _Static_assert (offsetof (tcbhead_t, __glibc_unused2) == 0x80,
while (0)
# define THREAD_GSCOPE_SET_FLAG() \
THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED)
# define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */