2005-04-17 06:20:36 +08:00
|
|
|
#ifndef _LINUX_SCHED_H
|
|
|
|
|
#define _LINUX_SCHED_H
|
|
|
|
|
|
2006-04-27 07:12:56 +08:00
|
|
|
/*
|
|
|
|
|
* cloning flags:
|
|
|
|
|
*/
|
|
|
|
|
#define CSIGNAL 0x000000ff /* signal mask to be sent at exit */
|
|
|
|
|
#define CLONE_VM 0x00000100 /* set if VM shared between processes */
|
|
|
|
|
#define CLONE_FS 0x00000200 /* set if fs info shared between processes */
|
|
|
|
|
#define CLONE_FILES 0x00000400 /* set if open files shared between processes */
|
|
|
|
|
#define CLONE_SIGHAND 0x00000800 /* set if signal handlers and blocked signals shared */
|
|
|
|
|
#define CLONE_PTRACE 0x00002000 /* set if we want to let tracing continue on the child too */
|
|
|
|
|
#define CLONE_VFORK 0x00004000 /* set if the parent wants the child to wake it up on mm_release */
|
|
|
|
|
#define CLONE_PARENT 0x00008000 /* set if we want to have the same parent as the cloner */
|
|
|
|
|
#define CLONE_THREAD 0x00010000 /* Same thread group? */
|
|
|
|
|
#define CLONE_NEWNS 0x00020000 /* New namespace group? */
|
|
|
|
|
#define CLONE_SYSVSEM 0x00040000 /* share system V SEM_UNDO semantics */
|
|
|
|
|
#define CLONE_SETTLS 0x00080000 /* create a new TLS for the child */
|
|
|
|
|
#define CLONE_PARENT_SETTID 0x00100000 /* set the TID in the parent */
|
|
|
|
|
#define CLONE_CHILD_CLEARTID 0x00200000 /* clear the TID in the child */
|
|
|
|
|
#define CLONE_DETACHED 0x00400000 /* Unused, ignored */
|
|
|
|
|
#define CLONE_UNTRACED 0x00800000 /* set if the tracing process can't force CLONE_PTRACE on this clone */
|
|
|
|
|
#define CLONE_CHILD_SETTID 0x01000000 /* set the TID in the child */
|
|
|
|
|
#define CLONE_STOPPED 0x02000000 /* Start in stopped state */
|
2006-10-02 17:18:17 +08:00
|
|
|
#define CLONE_NEWUTS 0x04000000 /* New utsname group? */
|
2006-10-02 17:18:19 +08:00
|
|
|
#define CLONE_NEWIPC 0x08000000 /* New ipcs */
|
2007-07-16 14:41:01 +08:00
|
|
|
#define CLONE_NEWUSER 0x10000000 /* New user namespace */
|
2007-10-19 14:40:10 +08:00
|
|
|
#define CLONE_NEWPID 0x20000000 /* New pid namespace */
|
2007-09-28 08:10:06 +08:00
|
|
|
#define CLONE_NEWNET 0x40000000 /* New network namespace */
|
2008-01-24 15:54:47 +08:00
|
|
|
#define CLONE_IO 0x80000000 /* Clone io context */
|
2006-04-27 07:12:56 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Scheduling policies
|
|
|
|
|
*/
|
|
|
|
|
#define SCHED_NORMAL 0
|
|
|
|
|
#define SCHED_FIFO 1
|
|
|
|
|
#define SCHED_RR 2
|
|
|
|
|
#define SCHED_BATCH 3
|
2007-07-10 00:51:57 +08:00
|
|
|
/* SCHED_ISO: reserved but not implemented yet */
|
|
|
|
|
#define SCHED_IDLE 5
|
2009-06-15 23:17:47 +08:00
|
|
|
/* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */
|
|
|
|
|
#define SCHED_RESET_ON_FORK 0x40000000
|
2006-04-27 07:12:56 +08:00
|
|
|
|
2006-04-25 21:54:40 +08:00
|
|
|
#ifdef __KERNEL__
|
2006-04-27 07:12:56 +08:00
|
|
|
|
|
|
|
|
struct sched_param {
|
|
|
|
|
int sched_priority;
|
|
|
|
|
};
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/param.h> /* for HZ */
|
|
|
|
|
|
|
|
|
|
#include <linux/capability.h>
|
|
|
|
|
#include <linux/threads.h>
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
|
#include <linux/types.h>
|
|
|
|
|
#include <linux/timex.h>
|
|
|
|
|
#include <linux/jiffies.h>
|
|
|
|
|
#include <linux/rbtree.h>
|
|
|
|
|
#include <linux/thread_info.h>
|
|
|
|
|
#include <linux/cpumask.h>
|
|
|
|
|
#include <linux/errno.h>
|
|
|
|
|
#include <linux/nodemask.h>
|
2007-10-16 16:24:43 +08:00
|
|
|
#include <linux/mm_types.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
#include <asm/system.h>
|
|
|
|
|
#include <asm/page.h>
|
|
|
|
|
#include <asm/ptrace.h>
|
|
|
|
|
#include <asm/cputime.h>
|
|
|
|
|
|
|
|
|
|
#include <linux/smp.h>
|
|
|
|
|
#include <linux/sem.h>
|
|
|
|
|
#include <linux/signal.h>
|
2009-03-30 07:50:06 +08:00
|
|
|
#include <linux/path.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/compiler.h>
|
|
|
|
|
#include <linux/completion.h>
|
|
|
|
|
#include <linux/pid.h>
|
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
|
#include <linux/topology.h>
|
2007-10-17 14:25:50 +08:00
|
|
|
#include <linux/proportions.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/seccomp.h>
|
2006-01-08 17:01:37 +08:00
|
|
|
#include <linux/rcupdate.h>
|
2009-04-15 02:17:16 +08:00
|
|
|
#include <linux/rculist.h>
|
2006-06-27 17:54:53 +08:00
|
|
|
#include <linux/rtmutex.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2006-04-25 21:54:40 +08:00
|
|
|
#include <linux/time.h>
|
|
|
|
|
#include <linux/param.h>
|
|
|
|
|
#include <linux/resource.h>
|
|
|
|
|
#include <linux/timer.h>
|
|
|
|
|
#include <linux/hrtimer.h>
|
2006-12-10 18:19:19 +08:00
|
|
|
#include <linux/task_io_accounting.h>
|
2007-10-15 23:00:14 +08:00
|
|
|
#include <linux/kobject.h>
|
2008-01-26 04:08:34 +08:00
|
|
|
#include <linux/latencytop.h>
|
2008-08-13 23:20:04 +08:00
|
|
|
#include <linux/cred.h>
|
2006-04-25 21:54:40 +08:00
|
|
|
|
|
|
|
|
#include <asm/processor.h>
|
2005-09-07 06:16:49 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
struct exec_domain;
|
2006-06-27 17:54:58 +08:00
|
|
|
struct futex_pi_state;
|
2008-01-26 04:08:34 +08:00
|
|
|
struct robust_list_head;
|
2010-02-23 15:55:42 +08:00
|
|
|
struct bio_list;
|
2009-03-30 07:50:06 +08:00
|
|
|
struct fs_struct;
|
perf: Do the big rename: Performance Counters -> Performance Events
Bye-bye Performance Counters, welcome Performance Events!
In the past few months the perfcounters subsystem has grown out its
initial role of counting hardware events, and has become (and is
becoming) a much broader generic event enumeration, reporting, logging,
monitoring, analysis facility.
Naming its core object 'perf_counter' and naming the subsystem
'perfcounters' has become more and more of a misnomer. With pending
code like hw-breakpoints support the 'counter' name is less and
less appropriate.
All in one, we've decided to rename the subsystem to 'performance
events' and to propagate this rename through all fields, variables
and API names. (in an ABI compatible fashion)
The word 'event' is also a bit shorter than 'counter' - which makes
it slightly more convenient to write/handle as well.
Thanks goes to Stephane Eranian who first observed this misnomer and
suggested a rename.
User-space tooling and ABI compatibility is not affected - this patch
should be function-invariant. (Also, defconfigs were not touched to
keep the size down.)
This patch has been generated via the following script:
FILES=$(find * -type f | grep -vE 'oprofile|[^K]config')
sed -i \
-e 's/PERF_EVENT_/PERF_RECORD_/g' \
-e 's/PERF_COUNTER/PERF_EVENT/g' \
-e 's/perf_counter/perf_event/g' \
-e 's/nb_counters/nb_events/g' \
-e 's/swcounter/swevent/g' \
-e 's/tpcounter_event/tp_event/g' \
$FILES
for N in $(find . -name perf_counter.[ch]); do
M=$(echo $N | sed 's/perf_counter/perf_event/g')
mv $N $M
done
FILES=$(find . -name perf_event.*)
sed -i \
-e 's/COUNTER_MASK/REG_MASK/g' \
-e 's/COUNTER/EVENT/g' \
-e 's/\<event\>/event_id/g' \
-e 's/counter/event/g' \
-e 's/Counter/Event/g' \
$FILES
... to keep it as correct as possible. This script can also be
used by anyone who has pending perfcounters patches - it converts
a Linux kernel tree over to the new naming. We tried to time this
change to the point in time where the amount of pending patches
is the smallest: the end of the merge window.
Namespace clashes were fixed up in a preparatory patch - and some
stylistic fallout will be fixed up in a subsequent patch.
( NOTE: 'counters' are still the proper terminology when we deal
with hardware registers - and these sed scripts are a bit
over-eager in renaming them. I've undone some of that, but
in case there's something left where 'counter' would be
better than 'event' we can undo that on an individual basis
instead of touching an otherwise nicely automated patch. )
Suggested-by: Stephane Eranian <eranian@google.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Paul Mackerras <paulus@samba.org>
Reviewed-by: Arjan van de Ven <arjan@linux.intel.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Howells <dhowells@redhat.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: <linux-arch@vger.kernel.org>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-21 18:02:48 +08:00
|
|
|
struct perf_event_context;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* List of flags we want to share for kernel threads,
|
|
|
|
|
* if only because they are not used by them anyway.
|
|
|
|
|
*/
|
|
|
|
|
#define CLONE_KERNEL (CLONE_FS | CLONE_FILES | CLONE_SIGHAND)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* These are the constant used to fake the fixed-point load-average
|
|
|
|
|
* counting. Some notes:
|
|
|
|
|
* - 11 bit fractions expand to 22 bits by the multiplies: this gives
|
|
|
|
|
* a load-average precision of 10 bits integer + 11 bits fractional
|
|
|
|
|
* - if you want to count load-averages more often, you need more
|
|
|
|
|
* precision, or rounding will get you. With 2-second counting freq,
|
|
|
|
|
* the EXP_n values would be 1981, 2034 and 2043 if still using only
|
|
|
|
|
* 11 bit fractions.
|
|
|
|
|
*/
|
|
|
|
|
extern unsigned long avenrun[]; /* Load averages */
|
2009-05-03 02:08:52 +08:00
|
|
|
extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
#define FSHIFT 11 /* nr of bits of precision */
|
|
|
|
|
#define FIXED_1 (1<<FSHIFT) /* 1.0 as fixed-point */
|
2007-10-08 07:17:38 +08:00
|
|
|
#define LOAD_FREQ (5*HZ+1) /* 5 sec intervals */
|
2005-04-17 06:20:36 +08:00
|
|
|
#define EXP_1 1884 /* 1/exp(5sec/1min) as fixed-point */
|
|
|
|
|
#define EXP_5 2014 /* 1/exp(5sec/5min) */
|
|
|
|
|
#define EXP_15 2037 /* 1/exp(5sec/15min) */
|
|
|
|
|
|
|
|
|
|
#define CALC_LOAD(load,exp,n) \
|
|
|
|
|
load *= exp; \
|
|
|
|
|
load += n*(FIXED_1-exp); \
|
|
|
|
|
load >>= FSHIFT;
|
|
|
|
|
|
|
|
|
|
extern unsigned long total_forks;
|
|
|
|
|
extern int nr_threads;
|
|
|
|
|
DECLARE_PER_CPU(unsigned long, process_counts);
|
|
|
|
|
extern int nr_processes(void);
|
|
|
|
|
extern unsigned long nr_running(void);
|
|
|
|
|
extern unsigned long nr_uninterruptible(void);
|
|
|
|
|
extern unsigned long nr_iowait(void);
|
2009-09-22 08:04:08 +08:00
|
|
|
extern unsigned long nr_iowait_cpu(void);
|
|
|
|
|
extern unsigned long this_cpu_load(void);
|
|
|
|
|
|
|
|
|
|
|
2009-04-11 16:43:41 +08:00
|
|
|
extern void calc_global_load(void);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-01-23 08:01:40 +08:00
|
|
|
extern unsigned long get_parent_ip(unsigned long addr);
|
|
|
|
|
|
2007-07-10 00:52:00 +08:00
|
|
|
struct seq_file;
|
|
|
|
|
struct cfs_rq;
|
2007-10-15 23:00:14 +08:00
|
|
|
struct task_group;
|
2007-07-10 00:52:00 +08:00
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
|
|
extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m);
|
|
|
|
|
extern void proc_sched_set_task(struct task_struct *p);
|
|
|
|
|
extern void
|
2007-08-09 17:16:47 +08:00
|
|
|
print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq);
|
2007-07-10 00:52:00 +08:00
|
|
|
#else
|
|
|
|
|
static inline void
|
|
|
|
|
proc_sched_show_task(struct task_struct *p, struct seq_file *m)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
static inline void proc_sched_set_task(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
static inline void
|
2007-08-09 17:16:47 +08:00
|
|
|
print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
|
2007-07-10 00:52:00 +08:00
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-09-30 06:18:21 +08:00
|
|
|
/*
|
|
|
|
|
* Task state bitmask. NOTE! These bits are also
|
|
|
|
|
* encoded in fs/proc/array.c: get_task_state().
|
|
|
|
|
*
|
|
|
|
|
* We have two separate sets of flags: task->state
|
|
|
|
|
* is about runnability, while task->exit_state are
|
|
|
|
|
* about the task exiting. Confusing, but this way
|
|
|
|
|
* modifying one set can't modify the other one by
|
|
|
|
|
* mistake.
|
|
|
|
|
*/
|
2005-04-17 06:20:36 +08:00
|
|
|
#define TASK_RUNNING 0
|
|
|
|
|
#define TASK_INTERRUPTIBLE 1
|
|
|
|
|
#define TASK_UNINTERRUPTIBLE 2
|
2007-12-07 00:13:16 +08:00
|
|
|
#define __TASK_STOPPED 4
|
|
|
|
|
#define __TASK_TRACED 8
|
2005-09-30 06:18:21 +08:00
|
|
|
/* in tsk->exit_state */
|
|
|
|
|
#define EXIT_ZOMBIE 16
|
|
|
|
|
#define EXIT_DEAD 32
|
|
|
|
|
/* in tsk->state again */
|
2007-10-15 23:00:13 +08:00
|
|
|
#define TASK_DEAD 64
|
2007-12-07 00:13:16 +08:00
|
|
|
#define TASK_WAKEKILL 128
|
2009-09-15 20:43:03 +08:00
|
|
|
#define TASK_WAKING 256
|
2009-12-17 20:16:30 +08:00
|
|
|
#define TASK_STATE_MAX 512
|
2007-12-07 00:13:16 +08:00
|
|
|
|
2009-12-17 20:16:28 +08:00
|
|
|
#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKW"
|
2009-12-17 20:16:27 +08:00
|
|
|
|
2009-12-17 20:16:30 +08:00
|
|
|
extern char ___assert_task_state[1 - 2*!!(
|
|
|
|
|
sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)];
|
2007-12-07 00:13:16 +08:00
|
|
|
|
|
|
|
|
/* Convenience macros for the sake of set_task_state */
|
|
|
|
|
#define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
|
|
|
|
|
#define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED)
|
|
|
|
|
#define TASK_TRACED (TASK_WAKEKILL | __TASK_TRACED)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-12-06 23:55:25 +08:00
|
|
|
/* Convenience macros for the sake of wake_up */
|
|
|
|
|
#define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
|
2007-12-07 00:13:16 +08:00
|
|
|
#define TASK_ALL (TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED)
|
2007-12-06 23:55:25 +08:00
|
|
|
|
|
|
|
|
/* get_task_state() */
|
|
|
|
|
#define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \
|
2007-12-07 00:13:16 +08:00
|
|
|
TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
|
|
|
|
|
__TASK_TRACED)
|
2007-12-06 23:55:25 +08:00
|
|
|
|
2007-12-07 00:13:16 +08:00
|
|
|
#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0)
|
|
|
|
|
#define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0)
|
2007-12-06 23:55:25 +08:00
|
|
|
#define task_is_stopped_or_traced(task) \
|
2007-12-07 00:13:16 +08:00
|
|
|
((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
|
2007-12-06 23:55:25 +08:00
|
|
|
#define task_contributes_to_load(task) \
|
2009-04-09 08:45:12 +08:00
|
|
|
((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
|
2009-07-17 20:15:47 +08:00
|
|
|
(task->flags & PF_FREEZING) == 0)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
#define __set_task_state(tsk, state_value) \
|
|
|
|
|
do { (tsk)->state = (state_value); } while (0)
|
|
|
|
|
#define set_task_state(tsk, state_value) \
|
|
|
|
|
set_mb((tsk)->state, (state_value))
|
|
|
|
|
|
2005-09-13 16:25:14 +08:00
|
|
|
/*
|
|
|
|
|
* set_current_state() includes a barrier so that the write of current->state
|
|
|
|
|
* is correctly serialised wrt the caller's subsequent test of whether to
|
|
|
|
|
* actually sleep:
|
|
|
|
|
*
|
|
|
|
|
* set_current_state(TASK_UNINTERRUPTIBLE);
|
|
|
|
|
* if (do_i_need_to_sleep())
|
|
|
|
|
* schedule();
|
|
|
|
|
*
|
|
|
|
|
* If the caller does not need such serialisation then use __set_current_state()
|
|
|
|
|
*/
|
2005-04-17 06:20:36 +08:00
|
|
|
#define __set_current_state(state_value) \
|
|
|
|
|
do { current->state = (state_value); } while (0)
|
|
|
|
|
#define set_current_state(state_value) \
|
|
|
|
|
set_mb(current->state, (state_value))
|
|
|
|
|
|
|
|
|
|
/* Task command name length */
|
|
|
|
|
#define TASK_COMM_LEN 16
|
|
|
|
|
|
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This serializes "schedule()" and also protects
|
|
|
|
|
* the run-queue from deletions/modifications (but
|
|
|
|
|
* _adding_ to the beginning of the run-queue has
|
|
|
|
|
* a separate lock).
|
|
|
|
|
*/
|
|
|
|
|
extern rwlock_t tasklist_lock;
|
|
|
|
|
extern spinlock_t mmlist_lock;
|
|
|
|
|
|
2006-07-03 15:25:41 +08:00
|
|
|
struct task_struct;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2010-03-03 23:46:56 +08:00
|
|
|
#ifdef CONFIG_PROVE_RCU
|
|
|
|
|
extern int lockdep_tasklist_lock_is_held(void);
|
|
|
|
|
#endif /* #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
extern void sched_init(void);
|
|
|
|
|
extern void sched_init_smp(void);
|
2008-02-16 01:56:34 +08:00
|
|
|
extern asmlinkage void schedule_tail(struct task_struct *prev);
|
2006-07-03 15:25:41 +08:00
|
|
|
extern void init_idle(struct task_struct *idle, int cpu);
|
2007-07-10 00:51:58 +08:00
|
|
|
extern void init_idle_bootup_task(struct task_struct *idle);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-09-20 02:55:44 +08:00
|
|
|
extern int runqueue_is_locked(int cpu);
|
2008-11-10 22:39:30 +08:00
|
|
|
extern void task_rq_unlock_wait(struct task_struct *p);
|
2008-05-13 03:20:52 +08:00
|
|
|
|
2008-11-25 00:05:04 +08:00
|
|
|
extern cpumask_var_t nohz_cpu_mask;
|
2007-05-08 15:32:51 +08:00
|
|
|
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ)
|
|
|
|
|
extern int select_nohz_load_balancer(int cpu);
|
2009-04-16 14:46:41 +08:00
|
|
|
extern int get_nohz_load_balancer(void);
|
2007-05-08 15:32:51 +08:00
|
|
|
#else
|
|
|
|
|
static inline int select_nohz_load_balancer(int cpu)
|
|
|
|
|
{
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2006-12-07 12:35:59 +08:00
|
|
|
/*
|
2007-04-26 11:50:03 +08:00
|
|
|
* Only dump TASK_* tasks. (0 for all tasks)
|
2006-12-07 12:35:59 +08:00
|
|
|
*/
|
|
|
|
|
extern void show_state_filter(unsigned long state_filter);
|
|
|
|
|
|
|
|
|
|
static inline void show_state(void)
|
|
|
|
|
{
|
2007-04-26 11:50:03 +08:00
|
|
|
show_state_filter(0);
|
2006-12-07 12:35:59 +08:00
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
extern void show_regs(struct pt_regs *);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* TASK is a pointer to the task whose backtrace we want to see (or NULL for current
|
|
|
|
|
* task), SP is the stack pointer of the first frame that should be shown in the back
|
|
|
|
|
* trace (or NULL if the entire call-chain of the task should be shown).
|
|
|
|
|
*/
|
|
|
|
|
extern void show_stack(struct task_struct *task, unsigned long *sp);
|
|
|
|
|
|
|
|
|
|
void io_schedule(void);
|
|
|
|
|
long io_schedule_timeout(long timeout);
|
|
|
|
|
|
|
|
|
|
extern void cpu_init (void);
|
|
|
|
|
extern void trap_init(void);
|
|
|
|
|
extern void update_process_times(int user);
|
|
|
|
|
extern void scheduler_tick(void);
|
|
|
|
|
|
2008-01-26 04:08:02 +08:00
|
|
|
extern void sched_show_task(struct task_struct *p);
|
|
|
|
|
|
2005-09-07 06:16:27 +08:00
|
|
|
#ifdef CONFIG_DETECT_SOFTLOCKUP
|
2006-03-24 19:18:41 +08:00
|
|
|
extern void softlockup_tick(void);
|
2005-09-07 06:16:27 +08:00
|
|
|
extern void touch_softlockup_watchdog(void);
|
2010-01-28 06:25:22 +08:00
|
|
|
extern void touch_softlockup_watchdog_sync(void);
|
2007-05-08 15:28:05 +08:00
|
|
|
extern void touch_all_softlockup_watchdogs(void);
|
2009-01-13 13:15:17 +08:00
|
|
|
extern int proc_dosoftlockup_thresh(struct ctl_table *table, int write,
|
2009-09-24 06:57:19 +08:00
|
|
|
void __user *buffer,
|
2009-01-13 13:15:17 +08:00
|
|
|
size_t *lenp, loff_t *ppos);
|
2008-05-13 03:21:04 +08:00
|
|
|
extern unsigned int softlockup_panic;
|
2008-05-13 03:21:14 +08:00
|
|
|
extern int softlockup_thresh;
|
2005-09-07 06:16:27 +08:00
|
|
|
#else
|
2006-03-24 19:18:41 +08:00
|
|
|
static inline void softlockup_tick(void)
|
2005-09-07 06:16:27 +08:00
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
static inline void touch_softlockup_watchdog(void)
|
|
|
|
|
{
|
|
|
|
|
}
|
2010-01-28 06:25:22 +08:00
|
|
|
static inline void touch_softlockup_watchdog_sync(void)
|
|
|
|
|
{
|
|
|
|
|
}
|
2007-05-08 15:28:05 +08:00
|
|
|
static inline void touch_all_softlockup_watchdogs(void)
|
|
|
|
|
{
|
|
|
|
|
}
|
2005-09-07 06:16:27 +08:00
|
|
|
#endif
|
|
|
|
|
|
2009-01-16 03:08:40 +08:00
|
|
|
#ifdef CONFIG_DETECT_HUNG_TASK
|
|
|
|
|
extern unsigned int sysctl_hung_task_panic;
|
|
|
|
|
extern unsigned long sysctl_hung_task_check_count;
|
|
|
|
|
extern unsigned long sysctl_hung_task_timeout_secs;
|
|
|
|
|
extern unsigned long sysctl_hung_task_warnings;
|
|
|
|
|
extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write,
|
2009-09-24 06:57:19 +08:00
|
|
|
void __user *buffer,
|
2009-01-16 03:08:40 +08:00
|
|
|
size_t *lenp, loff_t *ppos);
|
|
|
|
|
#endif
|
2005-09-07 06:16:27 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Attach to any functions which should be ignored in wchan output. */
|
|
|
|
|
#define __sched __attribute__((__section__(".sched.text")))
|
2007-11-28 22:52:56 +08:00
|
|
|
|
|
|
|
|
/* Linker adds these: start and end of __sched functions */
|
|
|
|
|
extern char __sched_text_start[], __sched_text_end[];
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Is this address in the __sched functions? */
|
|
|
|
|
extern int in_sched_functions(unsigned long addr);
|
|
|
|
|
|
|
|
|
|
#define MAX_SCHEDULE_TIMEOUT LONG_MAX
|
2008-02-14 07:03:15 +08:00
|
|
|
extern signed long schedule_timeout(signed long timeout);
|
2005-09-10 15:27:21 +08:00
|
|
|
extern signed long schedule_timeout_interruptible(signed long timeout);
|
2007-12-07 00:59:46 +08:00
|
|
|
extern signed long schedule_timeout_killable(signed long timeout);
|
2005-09-10 15:27:21 +08:00
|
|
|
extern signed long schedule_timeout_uninterruptible(signed long timeout);
|
2005-04-17 06:20:36 +08:00
|
|
|
asmlinkage void schedule(void);
|
2009-01-12 21:01:47 +08:00
|
|
|
extern int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2006-10-02 17:18:06 +08:00
|
|
|
struct nsproxy;
|
2007-07-16 14:40:59 +08:00
|
|
|
struct user_namespace;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-07-01 02:41:23 +08:00
|
|
|
/*
|
|
|
|
|
* Default maximum number of active map areas, this limits the number of vmas
|
|
|
|
|
* per mm struct. Users can overwrite this number by sysctl but there is a
|
|
|
|
|
* problem.
|
|
|
|
|
*
|
|
|
|
|
* When a program's coredump is generated as ELF format, a section is created
|
|
|
|
|
* per a vma. In ELF, the number of sections is represented in unsigned short.
|
|
|
|
|
* This means the number of sections should be smaller than 65535 at coredump.
|
|
|
|
|
* Because the kernel adds some informative sections to a image of program at
|
|
|
|
|
* generating coredump, we need some margin. The number of extra sections is
|
|
|
|
|
* 1-3 now and depends on arch. We use "5" as safe margin, here.
|
|
|
|
|
*/
|
|
|
|
|
#define MAPCOUNT_ELF_CORE_MARGIN (5)
|
|
|
|
|
#define DEFAULT_MAX_MAP_COUNT (USHORT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
extern int sysctl_max_map_count;
|
|
|
|
|
|
|
|
|
|
#include <linux/aio.h>
|
|
|
|
|
|
2010-01-16 09:01:35 +08:00
|
|
|
#ifdef CONFIG_MMU
|
|
|
|
|
extern void arch_pick_mmap_layout(struct mm_struct *mm);
|
2005-04-17 06:20:36 +08:00
|
|
|
extern unsigned long
|
|
|
|
|
arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
|
|
|
|
|
unsigned long, unsigned long);
|
|
|
|
|
extern unsigned long
|
|
|
|
|
arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
|
|
|
|
|
unsigned long len, unsigned long pgoff,
|
|
|
|
|
unsigned long flags);
|
2005-06-22 08:14:49 +08:00
|
|
|
extern void arch_unmap_area(struct mm_struct *, unsigned long);
|
|
|
|
|
extern void arch_unmap_area_topdown(struct mm_struct *, unsigned long);
|
2010-01-16 09:01:35 +08:00
|
|
|
#else
|
|
|
|
|
static inline void arch_pick_mmap_layout(struct mm_struct *mm) {}
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-01-07 06:40:29 +08:00
|
|
|
|
2007-07-19 16:48:27 +08:00
|
|
|
extern void set_dumpable(struct mm_struct *mm, int value);
|
|
|
|
|
extern int get_dumpable(struct mm_struct *mm);
|
|
|
|
|
|
|
|
|
|
/* mm flags */
|
2007-07-19 16:48:28 +08:00
|
|
|
/* dumpable bits */
|
2007-07-19 16:48:27 +08:00
|
|
|
#define MMF_DUMPABLE 0 /* core dump is permitted */
|
|
|
|
|
#define MMF_DUMP_SECURELY 1 /* core file is readable only by root */
|
2009-09-22 08:01:57 +08:00
|
|
|
|
2007-07-19 16:48:28 +08:00
|
|
|
#define MMF_DUMPABLE_BITS 2
|
2009-09-22 08:01:57 +08:00
|
|
|
#define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1)
|
2007-07-19 16:48:28 +08:00
|
|
|
|
|
|
|
|
/* coredump filter bits */
|
|
|
|
|
#define MMF_DUMP_ANON_PRIVATE 2
|
|
|
|
|
#define MMF_DUMP_ANON_SHARED 3
|
|
|
|
|
#define MMF_DUMP_MAPPED_PRIVATE 4
|
|
|
|
|
#define MMF_DUMP_MAPPED_SHARED 5
|
2007-10-17 14:27:02 +08:00
|
|
|
#define MMF_DUMP_ELF_HEADERS 6
|
coredump_filter: add hugepage dumping
Presently hugepage's vma has a VM_RESERVED flag in order not to be
swapped. But a VM_RESERVED vma isn't core dumped because this flag is
often used for some kernel vmas (e.g. vmalloc, sound related).
Thus hugepages are never dumped and it can't be debugged easily. Many
developers want hugepages to be included into core-dump.
However, We can't read generic VM_RESERVED area because this area is often
IO mapping area. then these area reading may change device state. it is
definitly undesiable side-effect.
So adding a hugepage specific bit to the coredump filter is better. It
will be able to hugepage core dumping and doesn't cause any side-effect to
any i/o devices.
In additional, libhugetlb use hugetlb private mapping pages as anonymous
page. Then, hugepage private mapping pages should be core dumped by
default.
Then, /proc/[pid]/core_dump_filter has two new bits.
- bit 5 mean hugetlb private mapping pages are dumped or not. (default: yes)
- bit 6 mean hugetlb shared mapping pages are dumped or not. (default: no)
I tested by following method.
% ulimit -c unlimited
% ./crash_hugepage 50
% ./crash_hugepage 50 -p
% ls -lh
% gdb ./crash_hugepage core
%
% echo 0x43 > /proc/self/coredump_filter
% ./crash_hugepage 50
% ./crash_hugepage 50 -p
% ls -lh
% gdb ./crash_hugepage core
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#include <string.h>
#include "hugetlbfs.h"
int main(int argc, char** argv){
char* p;
int ch;
int mmap_flags = MAP_SHARED;
int fd;
int nr_pages;
while((ch = getopt(argc, argv, "p")) != -1) {
switch (ch) {
case 'p':
mmap_flags &= ~MAP_SHARED;
mmap_flags |= MAP_PRIVATE;
break;
default:
/* nothing*/
break;
}
}
argc -= optind;
argv += optind;
if (argc == 0){
printf("need # of pages\n");
exit(1);
}
nr_pages = atoi(argv[0]);
if (nr_pages < 2) {
printf("nr_pages must >2\n");
exit(1);
}
fd = hugetlbfs_unlinked_fd();
p = mmap(NULL, nr_pages * gethugepagesize(),
PROT_READ|PROT_WRITE, mmap_flags, fd, 0);
sleep(2);
*(p + gethugepagesize()) = 1; /* COW */
sleep(2);
/* crash! */
*(int*)0 = 1;
return 0;
}
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reviewed-by: Kawai Hidehiro <hidehiro.kawai.ez@hitachi.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: William Irwin <wli@holomorphy.com>
Cc: Adam Litke <agl@us.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-10-19 11:27:08 +08:00
|
|
|
#define MMF_DUMP_HUGETLB_PRIVATE 7
|
|
|
|
|
#define MMF_DUMP_HUGETLB_SHARED 8
|
2009-09-22 08:01:57 +08:00
|
|
|
|
2007-07-19 16:48:28 +08:00
|
|
|
#define MMF_DUMP_FILTER_SHIFT MMF_DUMPABLE_BITS
|
coredump_filter: add hugepage dumping
Presently hugepage's vma has a VM_RESERVED flag in order not to be
swapped. But a VM_RESERVED vma isn't core dumped because this flag is
often used for some kernel vmas (e.g. vmalloc, sound related).
Thus hugepages are never dumped and it can't be debugged easily. Many
developers want hugepages to be included into core-dump.
However, We can't read generic VM_RESERVED area because this area is often
IO mapping area. then these area reading may change device state. it is
definitly undesiable side-effect.
So adding a hugepage specific bit to the coredump filter is better. It
will be able to hugepage core dumping and doesn't cause any side-effect to
any i/o devices.
In additional, libhugetlb use hugetlb private mapping pages as anonymous
page. Then, hugepage private mapping pages should be core dumped by
default.
Then, /proc/[pid]/core_dump_filter has two new bits.
- bit 5 mean hugetlb private mapping pages are dumped or not. (default: yes)
- bit 6 mean hugetlb shared mapping pages are dumped or not. (default: no)
I tested by following method.
% ulimit -c unlimited
% ./crash_hugepage 50
% ./crash_hugepage 50 -p
% ls -lh
% gdb ./crash_hugepage core
%
% echo 0x43 > /proc/self/coredump_filter
% ./crash_hugepage 50
% ./crash_hugepage 50 -p
% ls -lh
% gdb ./crash_hugepage core
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#include <string.h>
#include "hugetlbfs.h"
int main(int argc, char** argv){
char* p;
int ch;
int mmap_flags = MAP_SHARED;
int fd;
int nr_pages;
while((ch = getopt(argc, argv, "p")) != -1) {
switch (ch) {
case 'p':
mmap_flags &= ~MAP_SHARED;
mmap_flags |= MAP_PRIVATE;
break;
default:
/* nothing*/
break;
}
}
argc -= optind;
argv += optind;
if (argc == 0){
printf("need # of pages\n");
exit(1);
}
nr_pages = atoi(argv[0]);
if (nr_pages < 2) {
printf("nr_pages must >2\n");
exit(1);
}
fd = hugetlbfs_unlinked_fd();
p = mmap(NULL, nr_pages * gethugepagesize(),
PROT_READ|PROT_WRITE, mmap_flags, fd, 0);
sleep(2);
*(p + gethugepagesize()) = 1; /* COW */
sleep(2);
/* crash! */
*(int*)0 = 1;
return 0;
}
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reviewed-by: Kawai Hidehiro <hidehiro.kawai.ez@hitachi.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: William Irwin <wli@holomorphy.com>
Cc: Adam Litke <agl@us.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-10-19 11:27:08 +08:00
|
|
|
#define MMF_DUMP_FILTER_BITS 7
|
2007-07-19 16:48:28 +08:00
|
|
|
#define MMF_DUMP_FILTER_MASK \
|
|
|
|
|
(((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT)
|
|
|
|
|
#define MMF_DUMP_FILTER_DEFAULT \
|
coredump_filter: add hugepage dumping
Presently hugepage's vma has a VM_RESERVED flag in order not to be
swapped. But a VM_RESERVED vma isn't core dumped because this flag is
often used for some kernel vmas (e.g. vmalloc, sound related).
Thus hugepages are never dumped and it can't be debugged easily. Many
developers want hugepages to be included into core-dump.
However, We can't read generic VM_RESERVED area because this area is often
IO mapping area. then these area reading may change device state. it is
definitly undesiable side-effect.
So adding a hugepage specific bit to the coredump filter is better. It
will be able to hugepage core dumping and doesn't cause any side-effect to
any i/o devices.
In additional, libhugetlb use hugetlb private mapping pages as anonymous
page. Then, hugepage private mapping pages should be core dumped by
default.
Then, /proc/[pid]/core_dump_filter has two new bits.
- bit 5 mean hugetlb private mapping pages are dumped or not. (default: yes)
- bit 6 mean hugetlb shared mapping pages are dumped or not. (default: no)
I tested by following method.
% ulimit -c unlimited
% ./crash_hugepage 50
% ./crash_hugepage 50 -p
% ls -lh
% gdb ./crash_hugepage core
%
% echo 0x43 > /proc/self/coredump_filter
% ./crash_hugepage 50
% ./crash_hugepage 50 -p
% ls -lh
% gdb ./crash_hugepage core
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#include <string.h>
#include "hugetlbfs.h"
int main(int argc, char** argv){
char* p;
int ch;
int mmap_flags = MAP_SHARED;
int fd;
int nr_pages;
while((ch = getopt(argc, argv, "p")) != -1) {
switch (ch) {
case 'p':
mmap_flags &= ~MAP_SHARED;
mmap_flags |= MAP_PRIVATE;
break;
default:
/* nothing*/
break;
}
}
argc -= optind;
argv += optind;
if (argc == 0){
printf("need # of pages\n");
exit(1);
}
nr_pages = atoi(argv[0]);
if (nr_pages < 2) {
printf("nr_pages must >2\n");
exit(1);
}
fd = hugetlbfs_unlinked_fd();
p = mmap(NULL, nr_pages * gethugepagesize(),
PROT_READ|PROT_WRITE, mmap_flags, fd, 0);
sleep(2);
*(p + gethugepagesize()) = 1; /* COW */
sleep(2);
/* crash! */
*(int*)0 = 1;
return 0;
}
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reviewed-by: Kawai Hidehiro <hidehiro.kawai.ez@hitachi.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: William Irwin <wli@holomorphy.com>
Cc: Adam Litke <agl@us.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-10-19 11:27:08 +08:00
|
|
|
((1 << MMF_DUMP_ANON_PRIVATE) | (1 << MMF_DUMP_ANON_SHARED) |\
|
2008-10-19 11:28:23 +08:00
|
|
|
(1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF)
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS
|
|
|
|
|
# define MMF_DUMP_MASK_DEFAULT_ELF (1 << MMF_DUMP_ELF_HEADERS)
|
|
|
|
|
#else
|
|
|
|
|
# define MMF_DUMP_MASK_DEFAULT_ELF 0
|
|
|
|
|
#endif
|
2009-09-22 08:01:57 +08:00
|
|
|
/* leave room for more dump flags */
|
|
|
|
|
#define MMF_VM_MERGEABLE 16 /* KSM may merge identical pages */
|
|
|
|
|
|
|
|
|
|
#define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
|
2007-07-19 16:48:27 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sighand_struct {
|
|
|
|
|
atomic_t count;
|
|
|
|
|
struct k_sigaction action[_NSIG];
|
|
|
|
|
spinlock_t siglock;
|
2007-09-21 03:40:16 +08:00
|
|
|
wait_queue_head_t signalfd_wqh;
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
|
|
|
|
|
2006-06-25 20:49:24 +08:00
|
|
|
struct pacct_struct {
|
2006-06-25 20:49:25 +08:00
|
|
|
int ac_flag;
|
|
|
|
|
long ac_exitcode;
|
2006-06-25 20:49:24 +08:00
|
|
|
unsigned long ac_mem;
|
2006-06-25 20:49:26 +08:00
|
|
|
cputime_t ac_utime, ac_stime;
|
|
|
|
|
unsigned long ac_minflt, ac_majflt;
|
2006-06-25 20:49:24 +08:00
|
|
|
};
|
|
|
|
|
|
2009-07-29 18:15:26 +08:00
|
|
|
struct cpu_itimer {
|
|
|
|
|
cputime_t expires;
|
|
|
|
|
cputime_t incr;
|
2009-07-29 18:15:27 +08:00
|
|
|
u32 error;
|
|
|
|
|
u32 incr_error;
|
2009-07-29 18:15:26 +08:00
|
|
|
};
|
|
|
|
|
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
/**
|
|
|
|
|
* struct task_cputime - collected CPU time counts
|
|
|
|
|
* @utime: time spent in user mode, in &cputime_t units
|
|
|
|
|
* @stime: time spent in kernel mode, in &cputime_t units
|
|
|
|
|
* @sum_exec_runtime: total time spent on the CPU, in nanoseconds
|
2008-09-14 23:11:46 +08:00
|
|
|
*
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
* This structure groups together three kinds of CPU time that are
|
|
|
|
|
* tracked for threads and thread groups. Most things considering
|
|
|
|
|
* CPU time want to group these counts together and treat all three
|
|
|
|
|
* of them in parallel.
|
|
|
|
|
*/
|
|
|
|
|
struct task_cputime {
|
|
|
|
|
cputime_t utime;
|
|
|
|
|
cputime_t stime;
|
|
|
|
|
unsigned long long sum_exec_runtime;
|
|
|
|
|
};
|
|
|
|
|
/* Alternate field names when used to cache expirations. */
|
|
|
|
|
#define prof_exp stime
|
|
|
|
|
#define virt_exp utime
|
|
|
|
|
#define sched_exp sum_exec_runtime
|
|
|
|
|
|
2009-02-05 19:24:16 +08:00
|
|
|
#define INIT_CPUTIME \
|
|
|
|
|
(struct task_cputime) { \
|
|
|
|
|
.utime = cputime_zero, \
|
|
|
|
|
.stime = cputime_zero, \
|
|
|
|
|
.sum_exec_runtime = 0, \
|
|
|
|
|
}
|
|
|
|
|
|
2009-07-10 20:57:56 +08:00
|
|
|
/*
|
|
|
|
|
* Disable preemption until the scheduler is running.
|
|
|
|
|
* Reset by start_kernel()->sched_init()->init_idle().
|
2009-07-10 20:57:57 +08:00
|
|
|
*
|
|
|
|
|
* We include PREEMPT_ACTIVE to avoid cond_resched() from working
|
|
|
|
|
* before the scheduler is active -- see should_resched().
|
2009-07-10 20:57:56 +08:00
|
|
|
*/
|
2009-07-10 20:57:57 +08:00
|
|
|
#define INIT_PREEMPT_COUNT (1 + PREEMPT_ACTIVE)
|
2009-07-10 20:57:56 +08:00
|
|
|
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
/**
|
2009-02-05 19:24:16 +08:00
|
|
|
* struct thread_group_cputimer - thread group interval timer counts
|
|
|
|
|
* @cputime: thread group interval timers.
|
|
|
|
|
* @running: non-zero when there are timers running and
|
|
|
|
|
* @cputime receives updates.
|
|
|
|
|
* @lock: lock for fields in this struct.
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
*
|
|
|
|
|
* This structure contains the version of task_cputime, above, that is
|
2009-02-05 19:24:16 +08:00
|
|
|
* used for thread group CPU timer calculations.
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
*/
|
2009-02-05 19:24:16 +08:00
|
|
|
struct thread_group_cputimer {
|
|
|
|
|
struct task_cputime cputime;
|
|
|
|
|
int running;
|
|
|
|
|
spinlock_t lock;
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
};
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* NOTE! "signal_struct" does not have it's own
|
|
|
|
|
* locking, because a shared signal_struct always
|
|
|
|
|
* implies a shared sighand_struct, so locking
|
|
|
|
|
* sighand_struct is always a proper superset of
|
|
|
|
|
* the locking of signal_struct.
|
|
|
|
|
*/
|
|
|
|
|
struct signal_struct {
|
|
|
|
|
atomic_t count;
|
|
|
|
|
atomic_t live;
|
|
|
|
|
|
|
|
|
|
wait_queue_head_t wait_chldexit; /* for wait4() */
|
|
|
|
|
|
|
|
|
|
/* current thread group signal load-balancing target: */
|
2006-07-03 15:25:41 +08:00
|
|
|
struct task_struct *curr_target;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/* shared signal handling: */
|
|
|
|
|
struct sigpending shared_pending;
|
|
|
|
|
|
|
|
|
|
/* thread group exit support */
|
|
|
|
|
int group_exit_code;
|
|
|
|
|
/* overloaded:
|
|
|
|
|
* - notify group_exit_task when ->count is equal to notify_count
|
|
|
|
|
* - everyone except group_exit_task is stopped during signal delivery
|
|
|
|
|
* of fatal signals, group_exit_task processes the signal.
|
|
|
|
|
*/
|
|
|
|
|
int notify_count;
|
2008-08-01 20:18:04 +08:00
|
|
|
struct task_struct *group_exit_task;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/* thread group stop support, overloads group_exit_code too */
|
|
|
|
|
int group_stop_count;
|
|
|
|
|
unsigned int flags; /* see SIGNAL_* flags below */
|
|
|
|
|
|
|
|
|
|
/* POSIX.1b Interval Timers */
|
|
|
|
|
struct list_head posix_timers;
|
|
|
|
|
|
|
|
|
|
/* ITIMER_REAL timer for the process */
|
2006-01-10 12:52:34 +08:00
|
|
|
struct hrtimer real_timer;
|
2008-02-08 20:19:19 +08:00
|
|
|
struct pid *leader_pid;
|
2006-01-10 12:52:34 +08:00
|
|
|
ktime_t it_real_incr;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-07-29 18:15:26 +08:00
|
|
|
/*
|
|
|
|
|
* ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use
|
|
|
|
|
* CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these
|
|
|
|
|
* values are defined to 0 and 1 respectively
|
|
|
|
|
*/
|
|
|
|
|
struct cpu_itimer it[2];
|
2005-04-17 06:20:36 +08:00
|
|
|
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
/*
|
2009-02-05 19:24:16 +08:00
|
|
|
* Thread group totals for process CPU timers.
|
|
|
|
|
* See thread_group_cputimer(), et al, for details.
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
*/
|
2009-02-05 19:24:16 +08:00
|
|
|
struct thread_group_cputimer cputimer;
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
|
|
|
|
|
/* Earliest-expiration cache. */
|
|
|
|
|
struct task_cputime cputime_expires;
|
|
|
|
|
|
|
|
|
|
struct list_head cpu_timers[3];
|
|
|
|
|
|
2007-02-12 16:53:00 +08:00
|
|
|
struct pid *tty_old_pgrp;
|
2006-12-08 18:37:55 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* boolean value for session group leader */
|
|
|
|
|
int leader;
|
|
|
|
|
|
|
|
|
|
struct tty_struct *tty; /* NULL if no tty */
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Cumulative resource counters for dead threads in the group,
|
|
|
|
|
* and for reaped dead child processes forked by this group.
|
|
|
|
|
* Live threads maintain their own counters and add to these
|
|
|
|
|
* in __exit_signal, except for the group leader.
|
|
|
|
|
*/
|
2009-02-05 19:24:15 +08:00
|
|
|
cputime_t utime, stime, cutime, cstime;
|
2007-10-15 23:00:19 +08:00
|
|
|
cputime_t gtime;
|
|
|
|
|
cputime_t cgtime;
|
sched, cputime: Introduce thread_group_times()
This is a real fix for problem of utime/stime values decreasing
described in the thread:
http://lkml.org/lkml/2009/11/3/522
Now cputime is accounted in the following way:
- {u,s}time in task_struct are increased every time when the thread
is interrupted by a tick (timer interrupt).
- When a thread exits, its {u,s}time are added to signal->{u,s}time,
after adjusted by task_times().
- When all threads in a thread_group exits, accumulated {u,s}time
(and also c{u,s}time) in signal struct are added to c{u,s}time
in signal struct of the group's parent.
So {u,s}time in task struct are "raw" tick count, while
{u,s}time and c{u,s}time in signal struct are "adjusted" values.
And accounted values are used by:
- task_times(), to get cputime of a thread:
This function returns adjusted values that originates from raw
{u,s}time and scaled by sum_exec_runtime that accounted by CFS.
- thread_group_cputime(), to get cputime of a thread group:
This function returns sum of all {u,s}time of living threads in
the group, plus {u,s}time in the signal struct that is sum of
adjusted cputimes of all exited threads belonged to the group.
The problem is the return value of thread_group_cputime(),
because it is mixed sum of "raw" value and "adjusted" value:
group's {u,s}time = foreach(thread){{u,s}time} + exited({u,s}time)
This misbehavior can break {u,s}time monotonicity.
Assume that if there is a thread that have raw values greater
than adjusted values (e.g. interrupted by 1000Hz ticks 50 times
but only runs 45ms) and if it exits, cputime will decrease (e.g.
-5ms).
To fix this, we could do:
group's {u,s}time = foreach(t){task_times(t)} + exited({u,s}time)
But task_times() contains hard divisions, so applying it for
every thread should be avoided.
This patch fixes the above problem in the following way:
- Modify thread's exit (= __exit_signal()) not to use task_times().
It means {u,s}time in signal struct accumulates raw values instead
of adjusted values. As the result it makes thread_group_cputime()
to return pure sum of "raw" values.
- Introduce a new function thread_group_times(*task, *utime, *stime)
that converts "raw" values of thread_group_cputime() to "adjusted"
values, in same calculation procedure as task_times().
- Modify group's exit (= wait_task_zombie()) to use this introduced
thread_group_times(). It make c{u,s}time in signal struct to
have adjusted values like before this patch.
- Replace some thread_group_cputime() by thread_group_times().
This replacements are only applied where conveys the "adjusted"
cputime to users, and where already uses task_times() near by it.
(i.e. sys_times(), getrusage(), and /proc/<PID>/stat.)
This patch have a positive side effect:
- Before this patch, if a group contains many short-life threads
(e.g. runs 0.9ms and not interrupted by ticks), the group's
cputime could be invisible since thread's cputime was accumulated
after adjusted: imagine adjustment function as adj(ticks, runtime),
{adj(0, 0.9) + adj(0, 0.9) + ....} = {0 + 0 + ....} = 0.
After this patch it will not happen because the adjustment is
applied after accumulated.
v2:
- remove if()s, put new variables into signal_struct.
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: Spencer Candland <spencer@bluehost.com>
Cc: Americo Wang <xiyou.wangcong@gmail.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: Stanislaw Gruszka <sgruszka@redhat.com>
LKML-Reference: <4B162517.8040909@jp.fujitsu.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-02 16:28:07 +08:00
|
|
|
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
|
|
|
|
|
cputime_t prev_utime, prev_stime;
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
|
|
|
|
|
unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt;
|
2007-05-11 13:22:37 +08:00
|
|
|
unsigned long inblock, oublock, cinblock, coublock;
|
getrusage: fill ru_maxrss value
Make ->ru_maxrss value in struct rusage filled accordingly to rss hiwater
mark. This struct is filled as a parameter to getrusage syscall.
->ru_maxrss value is set to KBs which is the way it is done in BSD
systems. /usr/bin/time (gnu time) application converts ->ru_maxrss to KBs
which seems to be incorrect behavior. Maintainer of this util was
notified by me with the patch which corrects it and cc'ed.
To make this happen we extend struct signal_struct by two fields. The
first one is ->maxrss which we use to store rss hiwater of the task. The
second one is ->cmaxrss which we use to store highest rss hiwater of all
task childs. These values are used in k_getrusage() to actually fill
->ru_maxrss. k_getrusage() uses current rss hiwater value directly if mm
struct exists.
Note:
exec() clear mm->hiwater_rss, but doesn't clear sig->maxrss.
it is intetionally behavior. *BSD getrusage have exec() inheriting.
test programs
========================================================
getrusage.c
===========
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <signal.h>
#include <sys/mman.h>
#include "common.h"
#define err(str) perror(str), exit(1)
int main(int argc, char** argv)
{
int status;
printf("allocate 100MB\n");
consume(100);
printf("testcase1: fork inherit? \n");
printf(" expect: initial.self ~= child.self\n");
show_rusage("initial");
if (__fork()) {
wait(&status);
} else {
show_rusage("fork child");
_exit(0);
}
printf("\n");
printf("testcase2: fork inherit? (cont.) \n");
printf(" expect: initial.children ~= 100MB, but child.children = 0\n");
show_rusage("initial");
if (__fork()) {
wait(&status);
} else {
show_rusage("child");
_exit(0);
}
printf("\n");
printf("testcase3: fork + malloc \n");
printf(" expect: child.self ~= initial.self + 50MB\n");
show_rusage("initial");
if (__fork()) {
wait(&status);
} else {
printf("allocate +50MB\n");
consume(50);
show_rusage("fork child");
_exit(0);
}
printf("\n");
printf("testcase4: grandchild maxrss\n");
printf(" expect: post_wait.children ~= 300MB\n");
show_rusage("initial");
if (__fork()) {
wait(&status);
show_rusage("post_wait");
} else {
system("./child -n 0 -g 300");
_exit(0);
}
printf("\n");
printf("testcase5: zombie\n");
printf(" expect: pre_wait ~= initial, IOW the zombie process is not accounted.\n");
printf(" post_wait ~= 400MB, IOW wait() collect child's max_rss. \n");
show_rusage("initial");
if (__fork()) {
sleep(1); /* children become zombie */
show_rusage("pre_wait");
wait(&status);
show_rusage("post_wait");
} else {
system("./child -n 400");
_exit(0);
}
printf("\n");
printf("testcase6: SIG_IGN\n");
printf(" expect: initial ~= after_zombie (child's 500MB alloc should be ignored).\n");
show_rusage("initial");
signal(SIGCHLD, SIG_IGN);
if (__fork()) {
sleep(1); /* children become zombie */
show_rusage("after_zombie");
} else {
system("./child -n 500");
_exit(0);
}
printf("\n");
signal(SIGCHLD, SIG_DFL);
printf("testcase7: exec (without fork) \n");
printf(" expect: initial ~= exec \n");
show_rusage("initial");
execl("./child", "child", "-v", NULL);
return 0;
}
child.c
=======
#include <sys/types.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/resource.h>
#include "common.h"
int main(int argc, char** argv)
{
int status;
int c;
long consume_size = 0;
long grandchild_consume_size = 0;
int show = 0;
while ((c = getopt(argc, argv, "n:g:v")) != -1) {
switch (c) {
case 'n':
consume_size = atol(optarg);
break;
case 'v':
show = 1;
break;
case 'g':
grandchild_consume_size = atol(optarg);
break;
default:
break;
}
}
if (show)
show_rusage("exec");
if (consume_size) {
printf("child alloc %ldMB\n", consume_size);
consume(consume_size);
}
if (grandchild_consume_size) {
if (fork()) {
wait(&status);
} else {
printf("grandchild alloc %ldMB\n", grandchild_consume_size);
consume(grandchild_consume_size);
exit(0);
}
}
return 0;
}
common.c
========
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include <signal.h>
#include <sys/mman.h>
#include "common.h"
#define err(str) perror(str), exit(1)
void show_rusage(char *prefix)
{
int err, err2;
struct rusage rusage_self;
struct rusage rusage_children;
printf("%s: ", prefix);
err = getrusage(RUSAGE_SELF, &rusage_self);
if (!err)
printf("self %ld ", rusage_self.ru_maxrss);
err2 = getrusage(RUSAGE_CHILDREN, &rusage_children);
if (!err2)
printf("children %ld ", rusage_children.ru_maxrss);
printf("\n");
}
/* Some buggy OS need this worthless CPU waste. */
void make_pagefault(void)
{
void *addr;
int size = getpagesize();
int i;
for (i=0; i<1000; i++) {
addr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (addr == MAP_FAILED)
err("make_pagefault");
memset(addr, 0, size);
munmap(addr, size);
}
}
void consume(int mega)
{
size_t sz = mega * 1024 * 1024;
void *ptr;
ptr = malloc(sz);
memset(ptr, 0, sz);
make_pagefault();
}
pid_t __fork(void)
{
pid_t pid;
pid = fork();
make_pagefault();
return pid;
}
common.h
========
void show_rusage(char *prefix);
void make_pagefault(void);
void consume(int mega);
pid_t __fork(void);
FreeBSD result (expected result)
========================================================
allocate 100MB
testcase1: fork inherit?
expect: initial.self ~= child.self
initial: self 103492 children 0
fork child: self 103540 children 0
testcase2: fork inherit? (cont.)
expect: initial.children ~= 100MB, but child.children = 0
initial: self 103540 children 103540
child: self 103564 children 0
testcase3: fork + malloc
expect: child.self ~= initial.self + 50MB
initial: self 103564 children 103564
allocate +50MB
fork child: self 154860 children 0
testcase4: grandchild maxrss
expect: post_wait.children ~= 300MB
initial: self 103564 children 154860
grandchild alloc 300MB
post_wait: self 103564 children 308720
testcase5: zombie
expect: pre_wait ~= initial, IOW the zombie process is not accounted.
post_wait ~= 400MB, IOW wait() collect child's max_rss.
initial: self 103564 children 308720
child alloc 400MB
pre_wait: self 103564 children 308720
post_wait: self 103564 children 411312
testcase6: SIG_IGN
expect: initial ~= after_zombie (child's 500MB alloc should be ignored).
initial: self 103564 children 411312
child alloc 500MB
after_zombie: self 103624 children 411312
testcase7: exec (without fork)
expect: initial ~= exec
initial: self 103624 children 411312
exec: self 103624 children 411312
Linux result (actual test result)
========================================================
allocate 100MB
testcase1: fork inherit?
expect: initial.self ~= child.self
initial: self 102848 children 0
fork child: self 102572 children 0
testcase2: fork inherit? (cont.)
expect: initial.children ~= 100MB, but child.children = 0
initial: self 102876 children 102644
child: self 102572 children 0
testcase3: fork + malloc
expect: child.self ~= initial.self + 50MB
initial: self 102876 children 102644
allocate +50MB
fork child: self 153804 children 0
testcase4: grandchild maxrss
expect: post_wait.children ~= 300MB
initial: self 102876 children 153864
grandchild alloc 300MB
post_wait: self 102876 children 307536
testcase5: zombie
expect: pre_wait ~= initial, IOW the zombie process is not accounted.
post_wait ~= 400MB, IOW wait() collect child's max_rss.
initial: self 102876 children 307536
child alloc 400MB
pre_wait: self 102876 children 307536
post_wait: self 102876 children 410076
testcase6: SIG_IGN
expect: initial ~= after_zombie (child's 500MB alloc should be ignored).
initial: self 102876 children 410076
child alloc 500MB
after_zombie: self 102880 children 410076
testcase7: exec (without fork)
expect: initial ~= exec
initial: self 102880 children 410076
exec: self 102880 children 410076
Signed-off-by: Jiri Pirko <jpirko@redhat.com>
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-23 07:44:10 +08:00
|
|
|
unsigned long maxrss, cmaxrss;
|
2008-07-28 06:48:12 +08:00
|
|
|
struct task_io_accounting ioac;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-02-05 19:24:15 +08:00
|
|
|
/*
|
|
|
|
|
* Cumulative ns of schedule CPU time fo dead threads in the
|
|
|
|
|
* group, not including a zombie group leader, (This only differs
|
|
|
|
|
* from jiffies_to_ns(utime + stime) if sched_clock uses something
|
|
|
|
|
* other than jiffies.)
|
|
|
|
|
*/
|
|
|
|
|
unsigned long long sum_sched_runtime;
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* We don't bother to synchronize most readers of this at all,
|
|
|
|
|
* because there is no reader checking a limit that actually needs
|
|
|
|
|
* to get both rlim_cur and rlim_max atomically, and either one
|
|
|
|
|
* alone is a single word that can safely be read normally.
|
|
|
|
|
* getrlimit/setrlimit use task_lock(current->group_leader) to
|
|
|
|
|
* protect this instead of the siglock, because they really
|
|
|
|
|
* have no need to disable irqs.
|
|
|
|
|
*/
|
|
|
|
|
struct rlimit rlim[RLIM_NLIMITS];
|
|
|
|
|
|
2006-06-25 20:49:24 +08:00
|
|
|
#ifdef CONFIG_BSD_PROCESS_ACCT
|
|
|
|
|
struct pacct_struct pacct; /* per-process accounting information */
|
|
|
|
|
#endif
|
2006-07-14 15:24:44 +08:00
|
|
|
#ifdef CONFIG_TASKSTATS
|
|
|
|
|
struct taskstats *stats;
|
|
|
|
|
#endif
|
Audit: add TTY input auditing
Add TTY input auditing, used to audit system administrator's actions. This is
required by various security standards such as DCID 6/3 and PCI to provide
non-repudiation of administrator's actions and to allow a review of past
actions if the administrator seems to overstep their duties or if the system
becomes misconfigured for unknown reasons. These requirements do not make it
necessary to audit TTY output as well.
Compared to an user-space keylogger, this approach records TTY input using the
audit subsystem, correlated with other audit events, and it is completely
transparent to the user-space application (e.g. the console ioctls still
work).
TTY input auditing works on a higher level than auditing all system calls
within the session, which would produce an overwhelming amount of mostly
useless audit events.
Add an "audit_tty" attribute, inherited across fork (). Data read from TTYs
by process with the attribute is sent to the audit subsystem by the kernel.
The audit netlink interface is extended to allow modifying the audit_tty
attribute, and to allow sending explanatory audit events from user-space (for
example, a shell might send an event containing the final command, after the
interactive command-line editing and history expansion is performed, which
might be difficult to decipher from the TTY input alone).
Because the "audit_tty" attribute is inherited across fork (), it would be set
e.g. for sshd restarted within an audited session. To prevent this, the
audit_tty attribute is cleared when a process with no open TTY file
descriptors (e.g. after daemon startup) opens a TTY.
See https://www.redhat.com/archives/linux-audit/2007-June/msg00000.html for a
more detailed rationale document for an older version of this patch.
[akpm@linux-foundation.org: build fix]
Signed-off-by: Miloslav Trmac <mitr@redhat.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Alan Cox <alan@lxorguk.ukuu.org.uk>
Cc: Paul Fulghum <paulkf@microgate.com>
Cc: Casey Schaufler <casey@schaufler-ca.com>
Cc: Steve Grubb <sgrubb@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:40:56 +08:00
|
|
|
#ifdef CONFIG_AUDIT
|
|
|
|
|
unsigned audit_tty;
|
|
|
|
|
struct tty_audit_buf *tty_audit_buf;
|
|
|
|
|
#endif
|
oom: move oom_adj value from task_struct to signal_struct
Currently, OOM logic callflow is here.
__out_of_memory()
select_bad_process() for each task
badness() calculate badness of one task
oom_kill_process() search child
oom_kill_task() kill target task and mm shared tasks with it
example, process-A have two thread, thread-A and thread-B and it have very
fat memory and each thread have following oom_adj and oom_score.
thread-A: oom_adj = OOM_DISABLE, oom_score = 0
thread-B: oom_adj = 0, oom_score = very-high
Then, select_bad_process() select thread-B, but oom_kill_task() refuse
kill the task because thread-A have OOM_DISABLE. Thus __out_of_memory()
call select_bad_process() again. but select_bad_process() select the same
task. It mean kernel fall in livelock.
The fact is, select_bad_process() must select killable task. otherwise
OOM logic go into livelock.
And root cause is, oom_adj shouldn't be per-thread value. it should be
per-process value because OOM-killer kill a process, not thread. Thus
This patch moves oomkilladj (now more appropriately named oom_adj) from
struct task_struct to struct signal_struct. it naturally prevent
select_bad_process() choose wrong task.
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Paul Menage <menage@google.com>
Cc: David Rientjes <rientjes@google.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-22 08:03:13 +08:00
|
|
|
|
|
|
|
|
int oom_adj; /* OOM kill score adjustment (bit shift) */
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
|
|
|
|
|
2005-06-26 05:57:23 +08:00
|
|
|
/* Context switch must be unlocked if interrupts are to be enabled */
|
|
|
|
|
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
|
|
|
|
|
# define __ARCH_WANT_UNLOCKED_CTXSW
|
|
|
|
|
#endif
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* Bits in flags field of signal_struct.
|
|
|
|
|
*/
|
|
|
|
|
#define SIGNAL_STOP_STOPPED 0x00000001 /* job control stop in effect */
|
|
|
|
|
#define SIGNAL_STOP_DEQUEUED 0x00000002 /* stop signal dequeued */
|
|
|
|
|
#define SIGNAL_STOP_CONTINUED 0x00000004 /* SIGCONT since WCONTINUED reap */
|
|
|
|
|
#define SIGNAL_GROUP_EXIT 0x00000008 /* group exit in progress */
|
2008-04-30 15:52:44 +08:00
|
|
|
/*
|
|
|
|
|
* Pending notifications to parent.
|
|
|
|
|
*/
|
|
|
|
|
#define SIGNAL_CLD_STOPPED 0x00000010
|
|
|
|
|
#define SIGNAL_CLD_CONTINUED 0x00000020
|
|
|
|
|
#define SIGNAL_CLD_MASK (SIGNAL_CLD_STOPPED|SIGNAL_CLD_CONTINUED)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-04-30 15:53:03 +08:00
|
|
|
#define SIGNAL_UNKILLABLE 0x00000040 /* for init: ignore fatal signals */
|
|
|
|
|
|
2008-02-05 14:27:24 +08:00
|
|
|
/* If true, all threads except ->group_exit_task have pending SIGKILL */
|
|
|
|
|
static inline int signal_group_exit(const struct signal_struct *sig)
|
|
|
|
|
{
|
|
|
|
|
return (sig->flags & SIGNAL_GROUP_EXIT) ||
|
|
|
|
|
(sig->group_exit_task != NULL);
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* Some day this will be a full-fledged user tracking system..
|
|
|
|
|
*/
|
|
|
|
|
struct user_struct {
|
|
|
|
|
atomic_t __count; /* reference count */
|
|
|
|
|
atomic_t processes; /* How many processes does this user have? */
|
|
|
|
|
atomic_t files; /* How many open files does this user have? */
|
|
|
|
|
atomic_t sigpending; /* How many pending signals does this user have? */
|
2006-06-02 04:10:59 +08:00
|
|
|
#ifdef CONFIG_INOTIFY_USER
|
[PATCH] inotify
inotify is intended to correct the deficiencies of dnotify, particularly
its inability to scale and its terrible user interface:
* dnotify requires the opening of one fd per each directory
that you intend to watch. This quickly results in too many
open files and pins removable media, preventing unmount.
* dnotify is directory-based. You only learn about changes to
directories. Sure, a change to a file in a directory affects
the directory, but you are then forced to keep a cache of
stat structures.
* dnotify's interface to user-space is awful. Signals?
inotify provides a more usable, simple, powerful solution to file change
notification:
* inotify's interface is a system call that returns a fd, not SIGIO.
You get a single fd, which is select()-able.
* inotify has an event that says "the filesystem that the item
you were watching is on was unmounted."
* inotify can watch directories or files.
Inotify is currently used by Beagle (a desktop search infrastructure),
Gamin (a FAM replacement), and other projects.
See Documentation/filesystems/inotify.txt.
Signed-off-by: Robert Love <rml@novell.com>
Cc: John McCutchan <ttb@tentacle.dhs.org>
Cc: Christoph Hellwig <hch@lst.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-07-13 05:06:03 +08:00
|
|
|
atomic_t inotify_watches; /* How many inotify watches does this user have? */
|
|
|
|
|
atomic_t inotify_devs; /* How many inotify devs does this user have opened? */
|
|
|
|
|
#endif
|
epoll: introduce resource usage limits
It has been thought that the per-user file descriptors limit would also
limit the resources that a normal user can request via the epoll
interface. Vegard Nossum reported a very simple program (a modified
version attached) that can make a normal user to request a pretty large
amount of kernel memory, well within the its maximum number of fds. To
solve such problem, default limits are now imposed, and /proc based
configuration has been introduced. A new directory has been created,
named /proc/sys/fs/epoll/ and inside there, there are two configuration
points:
max_user_instances = Maximum number of devices - per user
max_user_watches = Maximum number of "watched" fds - per user
The current default for "max_user_watches" limits the memory used by epoll
to store "watches", to 1/32 of the amount of the low RAM. As example, a
256MB 32bit machine, will have "max_user_watches" set to roughly 90000.
That should be enough to not break existing heavy epoll users. The
default value for "max_user_instances" is set to 128, that should be
enough too.
This also changes the userspace, because a new error code can now come out
from EPOLL_CTL_ADD (-ENOSPC). The EMFILE from epoll_create() was already
listed, so that should be ok.
[akpm@linux-foundation.org: use get_current_user()]
Signed-off-by: Davide Libenzi <davidel@xmailserver.org>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: <stable@kernel.org>
Cc: Cyrill Gorcunov <gorcunov@gmail.com>
Reported-by: Vegard Nossum <vegardno@ifi.uio.no>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-12-02 05:13:55 +08:00
|
|
|
#ifdef CONFIG_EPOLL
|
|
|
|
|
atomic_t epoll_watches; /* The number of file descriptors currently watched */
|
|
|
|
|
#endif
|
2007-10-17 14:30:09 +08:00
|
|
|
#ifdef CONFIG_POSIX_MQUEUE
|
2005-04-17 06:20:36 +08:00
|
|
|
/* protected by mq_lock */
|
|
|
|
|
unsigned long mq_bytes; /* How many bytes can be allocated to mqueue? */
|
2007-10-17 14:30:09 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned long locked_shm; /* How many pages of mlocked shm ? */
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_KEYS
|
|
|
|
|
struct key *uid_keyring; /* UID specific keyring */
|
|
|
|
|
struct key *session_keyring; /* UID's default session keyring */
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/* Hash table maintenance information */
|
2007-09-19 13:46:44 +08:00
|
|
|
struct hlist_node uidhash_node;
|
2005-04-17 06:20:36 +08:00
|
|
|
uid_t uid;
|
2008-10-16 05:38:45 +08:00
|
|
|
struct user_namespace *user_ns;
|
2007-10-15 23:00:09 +08:00
|
|
|
|
perf: Do the big rename: Performance Counters -> Performance Events
Bye-bye Performance Counters, welcome Performance Events!
In the past few months the perfcounters subsystem has grown out its
initial role of counting hardware events, and has become (and is
becoming) a much broader generic event enumeration, reporting, logging,
monitoring, analysis facility.
Naming its core object 'perf_counter' and naming the subsystem
'perfcounters' has become more and more of a misnomer. With pending
code like hw-breakpoints support the 'counter' name is less and
less appropriate.
All in one, we've decided to rename the subsystem to 'performance
events' and to propagate this rename through all fields, variables
and API names. (in an ABI compatible fashion)
The word 'event' is also a bit shorter than 'counter' - which makes
it slightly more convenient to write/handle as well.
Thanks goes to Stephane Eranian who first observed this misnomer and
suggested a rename.
User-space tooling and ABI compatibility is not affected - this patch
should be function-invariant. (Also, defconfigs were not touched to
keep the size down.)
This patch has been generated via the following script:
FILES=$(find * -type f | grep -vE 'oprofile|[^K]config')
sed -i \
-e 's/PERF_EVENT_/PERF_RECORD_/g' \
-e 's/PERF_COUNTER/PERF_EVENT/g' \
-e 's/perf_counter/perf_event/g' \
-e 's/nb_counters/nb_events/g' \
-e 's/swcounter/swevent/g' \
-e 's/tpcounter_event/tp_event/g' \
$FILES
for N in $(find . -name perf_counter.[ch]); do
M=$(echo $N | sed 's/perf_counter/perf_event/g')
mv $N $M
done
FILES=$(find . -name perf_event.*)
sed -i \
-e 's/COUNTER_MASK/REG_MASK/g' \
-e 's/COUNTER/EVENT/g' \
-e 's/\<event\>/event_id/g' \
-e 's/counter/event/g' \
-e 's/Counter/Event/g' \
$FILES
... to keep it as correct as possible. This script can also be
used by anyone who has pending perfcounters patches - it converts
a Linux kernel tree over to the new naming. We tried to time this
change to the point in time where the amount of pending patches
is the smallest: the end of the merge window.
Namespace clashes were fixed up in a preparatory patch - and some
stylistic fallout will be fixed up in a subsequent patch.
( NOTE: 'counters' are still the proper terminology when we deal
with hardware registers - and these sed scripts are a bit
over-eager in renaming them. I've undone some of that, but
in case there's something left where 'counter' would be
better than 'event' we can undo that on an individual basis
instead of touching an otherwise nicely automated patch. )
Suggested-by: Stephane Eranian <eranian@google.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Paul Mackerras <paulus@samba.org>
Reviewed-by: Arjan van de Ven <arjan@linux.intel.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Howells <dhowells@redhat.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: <linux-arch@vger.kernel.org>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-21 18:02:48 +08:00
|
|
|
#ifdef CONFIG_PERF_EVENTS
|
2009-05-15 21:19:27 +08:00
|
|
|
atomic_long_t locked_vm;
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
|
|
|
|
|
2007-11-02 20:47:53 +08:00
|
|
|
extern int uids_sysfs_init(void);
|
2007-10-15 23:00:14 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
extern struct user_struct *find_user(uid_t);
|
|
|
|
|
|
|
|
|
|
extern struct user_struct root_user;
|
|
|
|
|
#define INIT_USER (&root_user)
|
|
|
|
|
|
2008-11-14 07:39:16 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
struct backing_dev_info;
|
|
|
|
|
struct reclaim_state;
|
|
|
|
|
|
2006-07-14 15:24:38 +08:00
|
|
|
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sched_info {
|
|
|
|
|
/* cumulative counters */
|
2007-10-15 23:00:12 +08:00
|
|
|
unsigned long pcount; /* # of times run on this cpu */
|
2008-12-17 15:41:22 +08:00
|
|
|
unsigned long long run_delay; /* time spent waiting on a runqueue */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/* timestamps */
|
2007-07-10 00:52:00 +08:00
|
|
|
unsigned long long last_arrival,/* when we last ran on a cpu */
|
|
|
|
|
last_queued; /* when we were last queued to run */
|
2007-10-15 23:00:10 +08:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
|
/* BKL stats */
|
2007-10-19 03:32:56 +08:00
|
|
|
unsigned int bkl_count;
|
2007-10-15 23:00:10 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
2006-07-14 15:24:38 +08:00
|
|
|
#endif /* defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2006-07-14 15:24:36 +08:00
|
|
|
#ifdef CONFIG_TASK_DELAY_ACCT
|
|
|
|
|
struct task_delay_info {
|
|
|
|
|
spinlock_t lock;
|
|
|
|
|
unsigned int flags; /* Private per-task flags */
|
|
|
|
|
|
|
|
|
|
/* For each stat XXX, add following, aligned appropriately
|
|
|
|
|
*
|
|
|
|
|
* struct timespec XXX_start, XXX_end;
|
|
|
|
|
* u64 XXX_delay;
|
|
|
|
|
* u32 XXX_count;
|
|
|
|
|
*
|
|
|
|
|
* Atomicity of updates to XXX_delay, XXX_count protected by
|
|
|
|
|
* single lock above (split into XXX_lock if contention is an issue).
|
|
|
|
|
*/
|
2006-07-14 15:24:37 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* XXX_count is incremented on every XXX operation, the delay
|
|
|
|
|
* associated with the operation is added to XXX_delay.
|
|
|
|
|
* XXX_delay contains the accumulated delay time in nanoseconds.
|
|
|
|
|
*/
|
|
|
|
|
struct timespec blkio_start, blkio_end; /* Shared by blkio, swapin */
|
|
|
|
|
u64 blkio_delay; /* wait for sync block io completion */
|
|
|
|
|
u64 swapin_delay; /* wait for swapin block io completion */
|
|
|
|
|
u32 blkio_count; /* total count of the number of sync block */
|
|
|
|
|
/* io operations performed */
|
|
|
|
|
u32 swapin_count; /* total count of the number of swapin block */
|
|
|
|
|
/* io operations performed */
|
2008-07-25 16:48:52 +08:00
|
|
|
|
|
|
|
|
struct timespec freepages_start, freepages_end;
|
|
|
|
|
u64 freepages_delay; /* wait for memory reclaim */
|
|
|
|
|
u32 freepages_count; /* total count of memory reclaim */
|
2006-07-14 15:24:36 +08:00
|
|
|
};
|
2006-07-14 15:24:38 +08:00
|
|
|
#endif /* CONFIG_TASK_DELAY_ACCT */
|
|
|
|
|
|
|
|
|
|
static inline int sched_info_on(void)
|
|
|
|
|
{
|
|
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
|
return 1;
|
|
|
|
|
#elif defined(CONFIG_TASK_DELAY_ACCT)
|
|
|
|
|
extern int delayacct_on;
|
|
|
|
|
return delayacct_on;
|
|
|
|
|
#else
|
|
|
|
|
return 0;
|
2006-07-14 15:24:36 +08:00
|
|
|
#endif
|
2006-07-14 15:24:38 +08:00
|
|
|
}
|
2006-07-14 15:24:36 +08:00
|
|
|
|
2007-07-10 00:51:57 +08:00
|
|
|
enum cpu_idle_type {
|
|
|
|
|
CPU_IDLE,
|
|
|
|
|
CPU_NOT_IDLE,
|
|
|
|
|
CPU_NEWLY_IDLE,
|
|
|
|
|
CPU_MAX_IDLE_TYPES
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* sched-domains (multiprocessor balancing) declarations:
|
|
|
|
|
*/
|
2007-07-10 00:51:58 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Increase resolution of nice-level calculations:
|
|
|
|
|
*/
|
|
|
|
|
#define SCHED_LOAD_SHIFT 10
|
|
|
|
|
#define SCHED_LOAD_SCALE (1L << SCHED_LOAD_SHIFT)
|
|
|
|
|
|
2007-08-23 21:18:02 +08:00
|
|
|
#define SCHED_LOAD_SCALE_FUZZ SCHED_LOAD_SCALE
|
2005-04-17 06:20:36 +08:00
|
|
|
|
[PATCH] sched: implement smpnice
Problem:
The introduction of separate run queues per CPU has brought with it "nice"
enforcement problems that are best described by a simple example.
For the sake of argument suppose that on a single CPU machine with a
nice==19 hard spinner and a nice==0 hard spinner running that the nice==0
task gets 95% of the CPU and the nice==19 task gets 5% of the CPU. Now
suppose that there is a system with 2 CPUs and 2 nice==19 hard spinners and
2 nice==0 hard spinners running. The user of this system would be entitled
to expect that the nice==0 tasks each get 95% of a CPU and the nice==19
tasks only get 5% each. However, whether this expectation is met is pretty
much down to luck as there are four equally likely distributions of the
tasks to the CPUs that the load balancing code will consider to be balanced
with loads of 2.0 for each CPU. Two of these distributions involve one
nice==0 and one nice==19 task per CPU and in these circumstances the users
expectations will be met. The other two distributions both involve both
nice==0 tasks being on one CPU and both nice==19 being on the other CPU and
each task will get 50% of a CPU and the user's expectations will not be
met.
Solution:
The solution to this problem that is implemented in the attached patch is
to use weighted loads when determining if the system is balanced and, when
an imbalance is detected, to move an amount of weighted load between run
queues (as opposed to a number of tasks) to restore the balance. Once
again, the easiest way to explain why both of these measures are necessary
is to use a simple example. Suppose that (in a slight variation of the
above example) that we have a two CPU system with 4 nice==0 and 4 nice=19
hard spinning tasks running and that the 4 nice==0 tasks are on one CPU and
the 4 nice==19 tasks are on the other CPU. The weighted loads for the two
CPUs would be 4.0 and 0.2 respectively and the load balancing code would
move 2 tasks resulting in one CPU with a load of 2.0 and the other with
load of 2.2. If this was considered to be a big enough imbalance to
justify moving a task and that task was moved using the current
move_tasks() then it would move the highest priority task that it found and
this would result in one CPU with a load of 3.0 and the other with a load
of 1.2 which would result in the movement of a task in the opposite
direction and so on -- infinite loop. If, on the other hand, an amount of
load to be moved is calculated from the imbalance (in this case 0.1) and
move_tasks() skips tasks until it find ones whose contributions to the
weighted load are less than this amount it would move two of the nice==19
tasks resulting in a system with 2 nice==0 and 2 nice=19 on each CPU with
loads of 2.1 for each CPU.
One of the advantages of this mechanism is that on a system where all tasks
have nice==0 the load balancing calculations would be mathematically
identical to the current load balancing code.
Notes:
struct task_struct:
has a new field load_weight which (in a trade off of space for speed)
stores the contribution that this task makes to a CPU's weighted load when
it is runnable.
struct runqueue:
has a new field raw_weighted_load which is the sum of the load_weight
values for the currently runnable tasks on this run queue. This field
always needs to be updated when nr_running is updated so two new inline
functions inc_nr_running() and dec_nr_running() have been created to make
sure that this happens. This also offers a convenient way to optimize away
this part of the smpnice mechanism when CONFIG_SMP is not defined.
int try_to_wake_up():
in this function the value SCHED_LOAD_BALANCE is used to represent the load
contribution of a single task in various calculations in the code that
decides which CPU to put the waking task on. While this would be a valid
on a system where the nice values for the runnable tasks were distributed
evenly around zero it will lead to anomalous load balancing if the
distribution is skewed in either direction. To overcome this problem
SCHED_LOAD_SCALE has been replaced by the load_weight for the relevant task
or by the average load_weight per task for the queue in question (as
appropriate).
int move_tasks():
The modifications to this function were complicated by the fact that
active_load_balance() uses it to move exactly one task without checking
whether an imbalance actually exists. This precluded the simple
overloading of max_nr_move with max_load_move and necessitated the addition
of the latter as an extra argument to the function. The internal
implementation is then modified to move up to max_nr_move tasks and
max_load_move of weighted load. This slightly complicates the code where
move_tasks() is called and if ever active_load_balance() is changed to not
use move_tasks() the implementation of move_tasks() should be simplified
accordingly.
struct sched_group *find_busiest_group():
Similar to try_to_wake_up(), there are places in this function where
SCHED_LOAD_SCALE is used to represent the load contribution of a single
task and the same issues are created. A similar solution is adopted except
that it is now the average per task contribution to a group's load (as
opposed to a run queue) that is required. As this value is not directly
available from the group it is calculated on the fly as the queues in the
groups are visited when determining the busiest group.
A key change to this function is that it is no longer to scale down
*imbalance on exit as move_tasks() uses the load in its scaled form.
void set_user_nice():
has been modified to update the task's load_weight field when it's nice
value and also to ensure that its run queue's raw_weighted_load field is
updated if it was runnable.
From: "Siddha, Suresh B" <suresh.b.siddha@intel.com>
With smpnice, sched groups with highest priority tasks can mask the imbalance
between the other sched groups with in the same domain. This patch fixes some
of the listed down scenarios by not considering the sched groups which are
lightly loaded.
a) on a simple 4-way MP system, if we have one high priority and 4 normal
priority tasks, with smpnice we would like to see the high priority task
scheduled on one cpu, two other cpus getting one normal task each and the
fourth cpu getting the remaining two normal tasks. but with current
smpnice extra normal priority task keeps jumping from one cpu to another
cpu having the normal priority task. This is because of the
busiest_has_loaded_cpus, nr_loaded_cpus logic.. We are not including the
cpu with high priority task in max_load calculations but including that in
total and avg_load calcuations.. leading to max_load < avg_load and load
balance between cpus running normal priority tasks(2 Vs 1) will always show
imbalanace as one normal priority and the extra normal priority task will
keep moving from one cpu to another cpu having normal priority task..
b) 4-way system with HT (8 logical processors). Package-P0 T0 has a
highest priority task, T1 is idle. Package-P1 Both T0 and T1 have 1 normal
priority task each.. P2 and P3 are idle. With this patch, one of the
normal priority tasks on P1 will be moved to P2 or P3..
c) With the current weighted smp nice calculations, it doesn't always make
sense to look at the highest weighted runqueue in the busy group..
Consider a load balance scenario on a DP with HT system, with Package-0
containing one high priority and one low priority, Package-1 containing one
low priority(with other thread being idle).. Package-1 thinks that it need
to take the low priority thread from Package-0. And find_busiest_queue()
returns the cpu thread with highest priority task.. And ultimately(with
help of active load balance) we move high priority task to Package-1. And
same continues with Package-0 now, moving high priority task from package-1
to package-0.. Even without the presence of active load balance, load
balance will fail to balance the above scenario.. Fix find_busiest_queue
to use "imbalance" when it is lightly loaded.
[kernel@kolivas.org: sched: store weighted load on up]
[kernel@kolivas.org: sched: add discrete weighted cpu load function]
[suresh.b.siddha@intel.com: sched: remove dead code]
Signed-off-by: Peter Williams <pwil3058@bigpond.com.au>
Cc: "Siddha, Suresh B" <suresh.b.siddha@intel.com>
Cc: "Chen, Kenneth W" <kenneth.w.chen@intel.com>
Acked-by: Ingo Molnar <mingo@elte.hu>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Con Kolivas <kernel@kolivas.org>
Cc: John Hawkes <hawkes@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-27 17:54:34 +08:00
|
|
|
#ifdef CONFIG_SMP
|
2009-09-01 16:34:33 +08:00
|
|
|
#define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */
|
|
|
|
|
#define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */
|
|
|
|
|
#define SD_BALANCE_EXEC 0x0004 /* Balance on exec */
|
|
|
|
|
#define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 19:50:02 +08:00
|
|
|
#define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */
|
2009-09-01 16:34:33 +08:00
|
|
|
#define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */
|
2009-09-16 14:28:30 +08:00
|
|
|
#define SD_PREFER_LOCAL 0x0040 /* Prefer to keep tasks local to this domain */
|
2009-09-01 16:34:33 +08:00
|
|
|
#define SD_SHARE_CPUPOWER 0x0080 /* Domain members share cpu power */
|
|
|
|
|
#define SD_POWERSAVINGS_BALANCE 0x0100 /* Balance for power savings */
|
|
|
|
|
#define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share cpu pkg resources */
|
|
|
|
|
#define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */
|
sched: Merge select_task_rq_fair() and sched_balance_self()
The problem with wake_idle() is that is doesn't respect things like
cpu_power, which means it doesn't deal well with SMT nor the recent
RT interaction.
To cure this, it needs to do what sched_balance_self() does, which
leads to the possibility of merging select_task_rq_fair() and
sched_balance_self().
Modify sched_balance_self() to:
- update_shares() when walking up the domain tree,
(it only called it for the top domain, but it should
have done this anyway), which allows us to remove
this ugly bit from try_to_wake_up().
- do wake_affine() on the smallest domain that contains
both this (the waking) and the prev (the wakee) cpu for
WAKE invocations.
Then use the top-down balance steps it had to replace wake_idle().
This leads to the dissapearance of SD_WAKE_BALANCE and
SD_WAKE_IDLE_FAR, with SD_WAKE_IDLE replaced with SD_BALANCE_WAKE.
SD_WAKE_AFFINE needs SD_BALANCE_WAKE to be effective.
Touch all topology bits to replace the old with new SD flags --
platforms might need re-tuning, enabling SD_BALANCE_WAKE
conditionally on a NUMA distance seems like a good additional
feature, magny-core and small nehalem systems would want this
enabled, systems with slow interconnects would not.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-10 19:50:02 +08:00
|
|
|
|
2009-09-01 16:34:33 +08:00
|
|
|
#define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */
|
2006-06-27 17:54:42 +08:00
|
|
|
|
2008-12-19 01:56:09 +08:00
|
|
|
enum powersavings_balance_level {
|
|
|
|
|
POWERSAVINGS_BALANCE_NONE = 0, /* No power saving load balance */
|
|
|
|
|
POWERSAVINGS_BALANCE_BASIC, /* Fill one thread/core/package
|
|
|
|
|
* first for long running threads
|
|
|
|
|
*/
|
|
|
|
|
POWERSAVINGS_BALANCE_WAKEUP, /* Also bias task wakeups to semi-idle
|
|
|
|
|
* cpu package for power savings
|
|
|
|
|
*/
|
|
|
|
|
MAX_POWERSAVINGS_BALANCE_LEVELS
|
|
|
|
|
};
|
2006-10-03 16:14:09 +08:00
|
|
|
|
2008-12-19 01:56:02 +08:00
|
|
|
extern int sched_mc_power_savings, sched_smt_power_savings;
|
2006-10-03 16:14:09 +08:00
|
|
|
|
2008-12-19 01:56:02 +08:00
|
|
|
static inline int sd_balance_for_mc_power(void)
|
|
|
|
|
{
|
|
|
|
|
if (sched_smt_power_savings)
|
|
|
|
|
return SD_POWERSAVINGS_BALANCE;
|
2006-06-27 17:54:42 +08:00
|
|
|
|
2010-02-08 18:05:55 +08:00
|
|
|
if (!sched_mc_power_savings)
|
|
|
|
|
return SD_PREFER_SIBLING;
|
|
|
|
|
|
|
|
|
|
return 0;
|
2008-12-19 01:56:02 +08:00
|
|
|
}
|
2006-10-03 16:14:09 +08:00
|
|
|
|
2008-12-19 01:56:02 +08:00
|
|
|
static inline int sd_balance_for_package_power(void)
|
|
|
|
|
{
|
|
|
|
|
if (sched_mc_power_savings | sched_smt_power_savings)
|
|
|
|
|
return SD_POWERSAVINGS_BALANCE;
|
|
|
|
|
|
2009-09-01 16:34:33 +08:00
|
|
|
return SD_PREFER_SIBLING;
|
2008-12-19 01:56:02 +08:00
|
|
|
}
|
2006-06-27 17:54:42 +08:00
|
|
|
|
2008-12-19 01:56:47 +08:00
|
|
|
/*
|
|
|
|
|
* Optimise SD flags for power savings:
|
|
|
|
|
* SD_BALANCE_NEWIDLE helps agressive task consolidation and power savings.
|
|
|
|
|
* Keep default SD flags if sched_{smt,mc}_power_saving=0
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
static inline int sd_power_saving_flags(void)
|
|
|
|
|
{
|
|
|
|
|
if (sched_mc_power_savings | sched_smt_power_savings)
|
|
|
|
|
return SD_BALANCE_NEWIDLE;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
struct sched_group {
|
|
|
|
|
struct sched_group *next; /* Must be a circular list */
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* CPU power of this group, SCHED_LOAD_SCALE being max power for a
|
2009-09-01 16:34:39 +08:00
|
|
|
* single CPU.
|
2007-05-08 15:32:57 +08:00
|
|
|
*/
|
2009-09-01 16:34:39 +08:00
|
|
|
unsigned int cpu_power;
|
2008-11-25 00:05:04 +08:00
|
|
|
|
2009-05-19 15:22:19 +08:00
|
|
|
/*
|
|
|
|
|
* The CPUs this group covers.
|
|
|
|
|
*
|
|
|
|
|
* NOTE: this field is variable length. (Allocated dynamically
|
|
|
|
|
* by attaching extra space to the end of the structure,
|
|
|
|
|
* depending on how many CPUs the kernel has booted up with)
|
|
|
|
|
*
|
|
|
|
|
* It is also be embedded into static data structures at build
|
|
|
|
|
* time. (See 'struct static_sched_group' in kernel/sched.c)
|
|
|
|
|
*/
|
|
|
|
|
unsigned long cpumask[0];
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
|
|
|
|
|
2008-11-25 00:05:04 +08:00
|
|
|
static inline struct cpumask *sched_group_cpus(struct sched_group *sg)
|
|
|
|
|
{
|
2008-11-25 00:05:04 +08:00
|
|
|
return to_cpumask(sg->cpumask);
|
2008-11-25 00:05:04 +08:00
|
|
|
}
|
|
|
|
|
|
2008-04-15 13:04:23 +08:00
|
|
|
enum sched_domain_level {
|
|
|
|
|
SD_LV_NONE = 0,
|
|
|
|
|
SD_LV_SIBLING,
|
|
|
|
|
SD_LV_MC,
|
|
|
|
|
SD_LV_CPU,
|
|
|
|
|
SD_LV_NODE,
|
|
|
|
|
SD_LV_ALLNODES,
|
|
|
|
|
SD_LV_MAX
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct sched_domain_attr {
|
|
|
|
|
int relax_domain_level;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
#define SD_ATTR_INIT (struct sched_domain_attr) { \
|
|
|
|
|
.relax_domain_level = -1, \
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sched_domain {
|
|
|
|
|
/* These fields must be setup */
|
|
|
|
|
struct sched_domain *parent; /* top domain must be null terminated */
|
2006-10-03 16:14:08 +08:00
|
|
|
struct sched_domain *child; /* bottom domain must be null terminated */
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sched_group *groups; /* the balancing groups of the domain */
|
|
|
|
|
unsigned long min_interval; /* Minimum balance interval ms */
|
|
|
|
|
unsigned long max_interval; /* Maximum balance interval ms */
|
|
|
|
|
unsigned int busy_factor; /* less balancing by factor if busy */
|
|
|
|
|
unsigned int imbalance_pct; /* No balance until over watermark */
|
|
|
|
|
unsigned int cache_nice_tries; /* Leave cache hot tasks for # tries */
|
2005-06-26 05:57:13 +08:00
|
|
|
unsigned int busy_idx;
|
|
|
|
|
unsigned int idle_idx;
|
|
|
|
|
unsigned int newidle_idx;
|
|
|
|
|
unsigned int wake_idx;
|
2005-06-26 05:57:19 +08:00
|
|
|
unsigned int forkexec_idx;
|
2009-09-01 16:34:35 +08:00
|
|
|
unsigned int smt_gain;
|
2005-04-17 06:20:36 +08:00
|
|
|
int flags; /* See SD_* */
|
2008-04-15 13:04:23 +08:00
|
|
|
enum sched_domain_level level;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/* Runtime fields. */
|
|
|
|
|
unsigned long last_balance; /* init to jiffies. units in jiffies */
|
|
|
|
|
unsigned int balance_interval; /* initialise to 1. units in ms. */
|
|
|
|
|
unsigned int nr_balance_failed; /* initialise to 0 */
|
|
|
|
|
|
2008-06-27 19:41:35 +08:00
|
|
|
u64 last_update;
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
|
|
|
|
/* load_balance() stats */
|
2007-10-19 03:32:56 +08:00
|
|
|
unsigned int lb_count[CPU_MAX_IDLE_TYPES];
|
|
|
|
|
unsigned int lb_failed[CPU_MAX_IDLE_TYPES];
|
|
|
|
|
unsigned int lb_balanced[CPU_MAX_IDLE_TYPES];
|
|
|
|
|
unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES];
|
|
|
|
|
unsigned int lb_gained[CPU_MAX_IDLE_TYPES];
|
|
|
|
|
unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES];
|
|
|
|
|
unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES];
|
|
|
|
|
unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES];
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/* Active load balancing */
|
2007-10-19 03:32:56 +08:00
|
|
|
unsigned int alb_count;
|
|
|
|
|
unsigned int alb_failed;
|
|
|
|
|
unsigned int alb_pushed;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-06-26 05:57:20 +08:00
|
|
|
/* SD_BALANCE_EXEC stats */
|
2007-10-19 03:32:56 +08:00
|
|
|
unsigned int sbe_count;
|
|
|
|
|
unsigned int sbe_balanced;
|
|
|
|
|
unsigned int sbe_pushed;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-06-26 05:57:20 +08:00
|
|
|
/* SD_BALANCE_FORK stats */
|
2007-10-19 03:32:56 +08:00
|
|
|
unsigned int sbf_count;
|
|
|
|
|
unsigned int sbf_balanced;
|
|
|
|
|
unsigned int sbf_pushed;
|
2005-06-26 05:57:20 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* try_to_wake_up() stats */
|
2007-10-19 03:32:56 +08:00
|
|
|
unsigned int ttwu_wake_remote;
|
|
|
|
|
unsigned int ttwu_move_affine;
|
|
|
|
|
unsigned int ttwu_move_balance;
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
2008-10-09 17:35:51 +08:00
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
|
|
char *name;
|
|
|
|
|
#endif
|
2008-11-25 00:05:04 +08:00
|
|
|
|
2009-05-19 15:22:19 +08:00
|
|
|
/*
|
|
|
|
|
* Span of all CPUs in this domain.
|
|
|
|
|
*
|
|
|
|
|
* NOTE: this field is variable length. (Allocated dynamically
|
|
|
|
|
* by attaching extra space to the end of the structure,
|
|
|
|
|
* depending on how many CPUs the kernel has booted up with)
|
|
|
|
|
*
|
|
|
|
|
* It is also be embedded into static data structures at build
|
|
|
|
|
* time. (See 'struct static_sched_domain' in kernel/sched.c)
|
|
|
|
|
*/
|
|
|
|
|
unsigned long span[0];
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
|
|
|
|
|
2008-11-25 00:05:04 +08:00
|
|
|
static inline struct cpumask *sched_domain_span(struct sched_domain *sd)
|
|
|
|
|
{
|
2008-11-25 00:05:04 +08:00
|
|
|
return to_cpumask(sd->span);
|
2008-11-25 00:05:04 +08:00
|
|
|
}
|
|
|
|
|
|
2009-11-03 12:23:40 +08:00
|
|
|
extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
|
2008-04-15 13:04:23 +08:00
|
|
|
struct sched_domain_attr *dattr_new);
|
2007-10-19 14:40:20 +08:00
|
|
|
|
2009-11-03 12:23:40 +08:00
|
|
|
/* Allocate an array of sched domains, for partition_sched_domains(). */
|
|
|
|
|
cpumask_var_t *alloc_sched_domains(unsigned int ndoms);
|
|
|
|
|
void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms);
|
|
|
|
|
|
2008-12-19 04:30:23 +08:00
|
|
|
/* Test a flag in parent sched domain */
|
|
|
|
|
static inline int test_sd_parent(struct sched_domain *sd, int flag)
|
|
|
|
|
{
|
|
|
|
|
if (sd->parent && (sd->parent->flags & flag))
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2007-10-19 14:40:20 +08:00
|
|
|
|
2009-09-02 19:49:18 +08:00
|
|
|
unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu);
|
|
|
|
|
unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu);
|
|
|
|
|
|
2008-07-18 20:01:39 +08:00
|
|
|
#else /* CONFIG_SMP */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-07-18 20:01:39 +08:00
|
|
|
struct sched_domain_attr;
|
2007-07-26 19:40:43 +08:00
|
|
|
|
2008-07-18 20:01:39 +08:00
|
|
|
static inline void
|
2009-11-03 12:23:40 +08:00
|
|
|
partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
|
2008-07-18 20:01:39 +08:00
|
|
|
struct sched_domain_attr *dattr_new)
|
|
|
|
|
{
|
2007-07-26 19:40:43 +08:00
|
|
|
}
|
2008-07-18 20:01:39 +08:00
|
|
|
#endif /* !CONFIG_SMP */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-09-02 19:49:18 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
struct io_context; /* See blkdev.h */
|
|
|
|
|
|
|
|
|
|
|
2005-09-10 04:02:02 +08:00
|
|
|
#ifdef ARCH_HAS_PREFETCH_SWITCH_STACK
|
2006-07-03 15:25:41 +08:00
|
|
|
extern void prefetch_stack(struct task_struct *t);
|
2005-09-10 04:02:02 +08:00
|
|
|
#else
|
|
|
|
|
static inline void prefetch_stack(struct task_struct *t) { }
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
struct audit_context; /* See audit.c */
|
|
|
|
|
struct mempolicy;
|
2006-04-11 19:52:07 +08:00
|
|
|
struct pipe_inode_info;
|
2006-10-02 17:18:14 +08:00
|
|
|
struct uts_namespace;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-07-10 00:51:58 +08:00
|
|
|
struct rq;
|
|
|
|
|
struct sched_domain;
|
|
|
|
|
|
2009-09-15 01:55:44 +08:00
|
|
|
/*
|
|
|
|
|
* wake flags
|
|
|
|
|
*/
|
|
|
|
|
#define WF_SYNC 0x01 /* waker goes to sleep after wakup */
|
2009-09-15 02:02:34 +08:00
|
|
|
#define WF_FORK 0x02 /* child wakeup after fork */
|
2009-09-15 01:55:44 +08:00
|
|
|
|
2007-07-10 00:51:58 +08:00
|
|
|
struct sched_class {
|
2007-10-15 23:00:12 +08:00
|
|
|
const struct sched_class *next;
|
2007-07-10 00:51:58 +08:00
|
|
|
|
2010-01-21 04:58:57 +08:00
|
|
|
void (*enqueue_task) (struct rq *rq, struct task_struct *p, int wakeup,
|
|
|
|
|
bool head);
|
2007-08-09 17:16:48 +08:00
|
|
|
void (*dequeue_task) (struct rq *rq, struct task_struct *p, int sleep);
|
2007-10-15 23:00:08 +08:00
|
|
|
void (*yield_task) (struct rq *rq);
|
2007-07-10 00:51:58 +08:00
|
|
|
|
2009-09-15 01:55:44 +08:00
|
|
|
void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags);
|
2007-07-10 00:51:58 +08:00
|
|
|
|
2007-08-09 17:16:48 +08:00
|
|
|
struct task_struct * (*pick_next_task) (struct rq *rq);
|
2007-08-09 17:16:49 +08:00
|
|
|
void (*put_prev_task) (struct rq *rq, struct task_struct *p);
|
2007-07-10 00:51:58 +08:00
|
|
|
|
2007-10-25 00:23:51 +08:00
|
|
|
#ifdef CONFIG_SMP
|
2009-09-15 01:55:44 +08:00
|
|
|
int (*select_task_rq)(struct task_struct *p, int sd_flag, int flags);
|
2008-10-22 15:25:26 +08:00
|
|
|
|
2008-01-26 04:08:22 +08:00
|
|
|
void (*pre_schedule) (struct rq *this_rq, struct task_struct *task);
|
|
|
|
|
void (*post_schedule) (struct rq *this_rq);
|
2009-12-17 01:04:40 +08:00
|
|
|
void (*task_waking) (struct rq *this_rq, struct task_struct *task);
|
|
|
|
|
void (*task_woken) (struct rq *this_rq, struct task_struct *task);
|
2007-10-25 00:23:51 +08:00
|
|
|
|
2008-03-27 05:23:49 +08:00
|
|
|
void (*set_cpus_allowed)(struct task_struct *p,
|
2008-11-25 00:05:14 +08:00
|
|
|
const struct cpumask *newmask);
|
2008-01-26 04:08:18 +08:00
|
|
|
|
2008-06-05 03:04:05 +08:00
|
|
|
void (*rq_online)(struct rq *rq);
|
|
|
|
|
void (*rq_offline)(struct rq *rq);
|
2008-10-22 15:25:26 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
void (*set_curr_task) (struct rq *rq);
|
|
|
|
|
void (*task_tick) (struct rq *rq, struct task_struct *p, int queued);
|
2009-11-28 00:32:46 +08:00
|
|
|
void (*task_fork) (struct task_struct *p);
|
2008-01-26 04:08:22 +08:00
|
|
|
|
|
|
|
|
void (*switched_from) (struct rq *this_rq, struct task_struct *task,
|
|
|
|
|
int running);
|
|
|
|
|
void (*switched_to) (struct rq *this_rq, struct task_struct *task,
|
|
|
|
|
int running);
|
|
|
|
|
void (*prio_changed) (struct rq *this_rq, struct task_struct *task,
|
|
|
|
|
int oldprio, int running);
|
2008-03-01 04:21:01 +08:00
|
|
|
|
2009-12-09 16:32:03 +08:00
|
|
|
unsigned int (*get_rr_interval) (struct rq *rq,
|
|
|
|
|
struct task_struct *task);
|
2009-09-21 09:31:53 +08:00
|
|
|
|
2008-03-01 04:21:01 +08:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
sched: Remove the cfs_rq dependency from set_task_cpu()
In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.
The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.
However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.
To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.
The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.
This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.
This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.
This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-17 01:04:41 +08:00
|
|
|
void (*moved_group) (struct task_struct *p, int on_rq);
|
2008-03-01 04:21:01 +08:00
|
|
|
#endif
|
2007-07-10 00:51:58 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct load_weight {
|
|
|
|
|
unsigned long weight, inv_weight;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* CFS stats for a schedulable entity (task, task-group etc)
|
|
|
|
|
*
|
|
|
|
|
* Current field usage histogram:
|
|
|
|
|
*
|
|
|
|
|
* 4 se->block_start
|
|
|
|
|
* 4 se->run_node
|
|
|
|
|
* 4 se->sleep_start
|
|
|
|
|
* 6 se->load.weight
|
|
|
|
|
*/
|
|
|
|
|
struct sched_entity {
|
|
|
|
|
struct load_weight load; /* for load-balancing */
|
|
|
|
|
struct rb_node run_node;
|
2008-04-20 01:45:00 +08:00
|
|
|
struct list_head group_node;
|
2007-07-10 00:51:58 +08:00
|
|
|
unsigned int on_rq;
|
|
|
|
|
|
2007-08-02 23:41:40 +08:00
|
|
|
u64 exec_start;
|
|
|
|
|
u64 sum_exec_runtime;
|
2007-10-15 23:00:04 +08:00
|
|
|
u64 vruntime;
|
2007-08-28 18:53:24 +08:00
|
|
|
u64 prev_sum_exec_runtime;
|
2007-08-02 23:41:40 +08:00
|
|
|
|
2008-03-19 08:42:00 +08:00
|
|
|
u64 last_wakeup;
|
|
|
|
|
u64 avg_overlap;
|
|
|
|
|
|
2008-12-14 19:34:15 +08:00
|
|
|
u64 nr_migrations;
|
|
|
|
|
|
2009-01-16 20:36:06 +08:00
|
|
|
u64 start_runtime;
|
|
|
|
|
u64 avg_wakeup;
|
|
|
|
|
|
2007-08-02 23:41:40 +08:00
|
|
|
#ifdef CONFIG_SCHEDSTATS
|
2007-07-10 00:51:58 +08:00
|
|
|
u64 wait_start;
|
2007-08-02 23:41:40 +08:00
|
|
|
u64 wait_max;
|
2008-01-26 04:08:35 +08:00
|
|
|
u64 wait_count;
|
|
|
|
|
u64 wait_sum;
|
2009-07-21 02:26:58 +08:00
|
|
|
u64 iowait_count;
|
|
|
|
|
u64 iowait_sum;
|
2007-08-02 23:41:40 +08:00
|
|
|
|
2007-07-10 00:51:58 +08:00
|
|
|
u64 sleep_start;
|
|
|
|
|
u64 sleep_max;
|
2007-08-02 23:41:40 +08:00
|
|
|
s64 sum_sleep_runtime;
|
|
|
|
|
|
|
|
|
|
u64 block_start;
|
2007-07-10 00:51:58 +08:00
|
|
|
u64 block_max;
|
|
|
|
|
u64 exec_max;
|
2007-10-15 23:00:02 +08:00
|
|
|
u64 slice_max;
|
2007-10-15 23:00:18 +08:00
|
|
|
|
|
|
|
|
u64 nr_migrations_cold;
|
|
|
|
|
u64 nr_failed_migrations_affine;
|
|
|
|
|
u64 nr_failed_migrations_running;
|
|
|
|
|
u64 nr_failed_migrations_hot;
|
|
|
|
|
u64 nr_forced_migrations;
|
|
|
|
|
|
|
|
|
|
u64 nr_wakeups;
|
|
|
|
|
u64 nr_wakeups_sync;
|
|
|
|
|
u64 nr_wakeups_migrate;
|
|
|
|
|
u64 nr_wakeups_local;
|
|
|
|
|
u64 nr_wakeups_remote;
|
|
|
|
|
u64 nr_wakeups_affine;
|
|
|
|
|
u64 nr_wakeups_affine_attempts;
|
|
|
|
|
u64 nr_wakeups_passive;
|
|
|
|
|
u64 nr_wakeups_idle;
|
2007-08-02 23:41:40 +08:00
|
|
|
#endif
|
|
|
|
|
|
2007-07-10 00:51:58 +08:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
|
|
|
|
struct sched_entity *parent;
|
|
|
|
|
/* rq on which this entity is (to be) queued: */
|
|
|
|
|
struct cfs_rq *cfs_rq;
|
|
|
|
|
/* rq "owned" by this entity/group: */
|
|
|
|
|
struct cfs_rq *my_q;
|
|
|
|
|
#endif
|
|
|
|
|
};
|
2006-07-03 15:25:42 +08:00
|
|
|
|
2008-01-26 04:08:27 +08:00
|
|
|
struct sched_rt_entity {
|
|
|
|
|
struct list_head run_list;
|
2008-01-26 04:08:27 +08:00
|
|
|
unsigned long timeout;
|
2008-08-01 20:24:08 +08:00
|
|
|
unsigned int time_slice;
|
2008-01-26 04:08:30 +08:00
|
|
|
int nr_cpus_allowed;
|
|
|
|
|
|
2008-04-20 01:45:00 +08:00
|
|
|
struct sched_rt_entity *back;
|
2008-02-13 22:45:40 +08:00
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
2008-01-26 04:08:30 +08:00
|
|
|
struct sched_rt_entity *parent;
|
|
|
|
|
/* rq on which this entity is (to be) queued: */
|
|
|
|
|
struct rt_rq *rt_rq;
|
|
|
|
|
/* rq "owned" by this entity/group: */
|
|
|
|
|
struct rt_rq *my_q;
|
|
|
|
|
#endif
|
2008-01-26 04:08:27 +08:00
|
|
|
};
|
|
|
|
|
|
2009-08-28 06:00:12 +08:00
|
|
|
struct rcu_node;
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
struct task_struct {
|
|
|
|
|
volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */
|
2007-05-09 17:35:17 +08:00
|
|
|
void *stack;
|
2005-04-17 06:20:36 +08:00
|
|
|
atomic_t usage;
|
2007-05-08 15:23:41 +08:00
|
|
|
unsigned int flags; /* per process flags, defined below */
|
|
|
|
|
unsigned int ptrace;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-05-06 07:16:12 +08:00
|
|
|
int lock_depth; /* BKL lock depth */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
[PATCH] sched: implement smpnice
Problem:
The introduction of separate run queues per CPU has brought with it "nice"
enforcement problems that are best described by a simple example.
For the sake of argument suppose that on a single CPU machine with a
nice==19 hard spinner and a nice==0 hard spinner running that the nice==0
task gets 95% of the CPU and the nice==19 task gets 5% of the CPU. Now
suppose that there is a system with 2 CPUs and 2 nice==19 hard spinners and
2 nice==0 hard spinners running. The user of this system would be entitled
to expect that the nice==0 tasks each get 95% of a CPU and the nice==19
tasks only get 5% each. However, whether this expectation is met is pretty
much down to luck as there are four equally likely distributions of the
tasks to the CPUs that the load balancing code will consider to be balanced
with loads of 2.0 for each CPU. Two of these distributions involve one
nice==0 and one nice==19 task per CPU and in these circumstances the users
expectations will be met. The other two distributions both involve both
nice==0 tasks being on one CPU and both nice==19 being on the other CPU and
each task will get 50% of a CPU and the user's expectations will not be
met.
Solution:
The solution to this problem that is implemented in the attached patch is
to use weighted loads when determining if the system is balanced and, when
an imbalance is detected, to move an amount of weighted load between run
queues (as opposed to a number of tasks) to restore the balance. Once
again, the easiest way to explain why both of these measures are necessary
is to use a simple example. Suppose that (in a slight variation of the
above example) that we have a two CPU system with 4 nice==0 and 4 nice=19
hard spinning tasks running and that the 4 nice==0 tasks are on one CPU and
the 4 nice==19 tasks are on the other CPU. The weighted loads for the two
CPUs would be 4.0 and 0.2 respectively and the load balancing code would
move 2 tasks resulting in one CPU with a load of 2.0 and the other with
load of 2.2. If this was considered to be a big enough imbalance to
justify moving a task and that task was moved using the current
move_tasks() then it would move the highest priority task that it found and
this would result in one CPU with a load of 3.0 and the other with a load
of 1.2 which would result in the movement of a task in the opposite
direction and so on -- infinite loop. If, on the other hand, an amount of
load to be moved is calculated from the imbalance (in this case 0.1) and
move_tasks() skips tasks until it find ones whose contributions to the
weighted load are less than this amount it would move two of the nice==19
tasks resulting in a system with 2 nice==0 and 2 nice=19 on each CPU with
loads of 2.1 for each CPU.
One of the advantages of this mechanism is that on a system where all tasks
have nice==0 the load balancing calculations would be mathematically
identical to the current load balancing code.
Notes:
struct task_struct:
has a new field load_weight which (in a trade off of space for speed)
stores the contribution that this task makes to a CPU's weighted load when
it is runnable.
struct runqueue:
has a new field raw_weighted_load which is the sum of the load_weight
values for the currently runnable tasks on this run queue. This field
always needs to be updated when nr_running is updated so two new inline
functions inc_nr_running() and dec_nr_running() have been created to make
sure that this happens. This also offers a convenient way to optimize away
this part of the smpnice mechanism when CONFIG_SMP is not defined.
int try_to_wake_up():
in this function the value SCHED_LOAD_BALANCE is used to represent the load
contribution of a single task in various calculations in the code that
decides which CPU to put the waking task on. While this would be a valid
on a system where the nice values for the runnable tasks were distributed
evenly around zero it will lead to anomalous load balancing if the
distribution is skewed in either direction. To overcome this problem
SCHED_LOAD_SCALE has been replaced by the load_weight for the relevant task
or by the average load_weight per task for the queue in question (as
appropriate).
int move_tasks():
The modifications to this function were complicated by the fact that
active_load_balance() uses it to move exactly one task without checking
whether an imbalance actually exists. This precluded the simple
overloading of max_nr_move with max_load_move and necessitated the addition
of the latter as an extra argument to the function. The internal
implementation is then modified to move up to max_nr_move tasks and
max_load_move of weighted load. This slightly complicates the code where
move_tasks() is called and if ever active_load_balance() is changed to not
use move_tasks() the implementation of move_tasks() should be simplified
accordingly.
struct sched_group *find_busiest_group():
Similar to try_to_wake_up(), there are places in this function where
SCHED_LOAD_SCALE is used to represent the load contribution of a single
task and the same issues are created. A similar solution is adopted except
that it is now the average per task contribution to a group's load (as
opposed to a run queue) that is required. As this value is not directly
available from the group it is calculated on the fly as the queues in the
groups are visited when determining the busiest group.
A key change to this function is that it is no longer to scale down
*imbalance on exit as move_tasks() uses the load in its scaled form.
void set_user_nice():
has been modified to update the task's load_weight field when it's nice
value and also to ensure that its run queue's raw_weighted_load field is
updated if it was runnable.
From: "Siddha, Suresh B" <suresh.b.siddha@intel.com>
With smpnice, sched groups with highest priority tasks can mask the imbalance
between the other sched groups with in the same domain. This patch fixes some
of the listed down scenarios by not considering the sched groups which are
lightly loaded.
a) on a simple 4-way MP system, if we have one high priority and 4 normal
priority tasks, with smpnice we would like to see the high priority task
scheduled on one cpu, two other cpus getting one normal task each and the
fourth cpu getting the remaining two normal tasks. but with current
smpnice extra normal priority task keeps jumping from one cpu to another
cpu having the normal priority task. This is because of the
busiest_has_loaded_cpus, nr_loaded_cpus logic.. We are not including the
cpu with high priority task in max_load calculations but including that in
total and avg_load calcuations.. leading to max_load < avg_load and load
balance between cpus running normal priority tasks(2 Vs 1) will always show
imbalanace as one normal priority and the extra normal priority task will
keep moving from one cpu to another cpu having normal priority task..
b) 4-way system with HT (8 logical processors). Package-P0 T0 has a
highest priority task, T1 is idle. Package-P1 Both T0 and T1 have 1 normal
priority task each.. P2 and P3 are idle. With this patch, one of the
normal priority tasks on P1 will be moved to P2 or P3..
c) With the current weighted smp nice calculations, it doesn't always make
sense to look at the highest weighted runqueue in the busy group..
Consider a load balance scenario on a DP with HT system, with Package-0
containing one high priority and one low priority, Package-1 containing one
low priority(with other thread being idle).. Package-1 thinks that it need
to take the low priority thread from Package-0. And find_busiest_queue()
returns the cpu thread with highest priority task.. And ultimately(with
help of active load balance) we move high priority task to Package-1. And
same continues with Package-0 now, moving high priority task from package-1
to package-0.. Even without the presence of active load balance, load
balance will fail to balance the above scenario.. Fix find_busiest_queue
to use "imbalance" when it is lightly loaded.
[kernel@kolivas.org: sched: store weighted load on up]
[kernel@kolivas.org: sched: add discrete weighted cpu load function]
[suresh.b.siddha@intel.com: sched: remove dead code]
Signed-off-by: Peter Williams <pwil3058@bigpond.com.au>
Cc: "Siddha, Suresh B" <suresh.b.siddha@intel.com>
Cc: "Chen, Kenneth W" <kenneth.w.chen@intel.com>
Acked-by: Ingo Molnar <mingo@elte.hu>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Con Kolivas <kernel@kolivas.org>
Cc: John Hawkes <hawkes@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-27 17:54:34 +08:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
#ifdef __ARCH_WANT_UNLOCKED_CTXSW
|
2005-06-26 05:57:23 +08:00
|
|
|
int oncpu;
|
|
|
|
|
#endif
|
[PATCH] sched: implement smpnice
Problem:
The introduction of separate run queues per CPU has brought with it "nice"
enforcement problems that are best described by a simple example.
For the sake of argument suppose that on a single CPU machine with a
nice==19 hard spinner and a nice==0 hard spinner running that the nice==0
task gets 95% of the CPU and the nice==19 task gets 5% of the CPU. Now
suppose that there is a system with 2 CPUs and 2 nice==19 hard spinners and
2 nice==0 hard spinners running. The user of this system would be entitled
to expect that the nice==0 tasks each get 95% of a CPU and the nice==19
tasks only get 5% each. However, whether this expectation is met is pretty
much down to luck as there are four equally likely distributions of the
tasks to the CPUs that the load balancing code will consider to be balanced
with loads of 2.0 for each CPU. Two of these distributions involve one
nice==0 and one nice==19 task per CPU and in these circumstances the users
expectations will be met. The other two distributions both involve both
nice==0 tasks being on one CPU and both nice==19 being on the other CPU and
each task will get 50% of a CPU and the user's expectations will not be
met.
Solution:
The solution to this problem that is implemented in the attached patch is
to use weighted loads when determining if the system is balanced and, when
an imbalance is detected, to move an amount of weighted load between run
queues (as opposed to a number of tasks) to restore the balance. Once
again, the easiest way to explain why both of these measures are necessary
is to use a simple example. Suppose that (in a slight variation of the
above example) that we have a two CPU system with 4 nice==0 and 4 nice=19
hard spinning tasks running and that the 4 nice==0 tasks are on one CPU and
the 4 nice==19 tasks are on the other CPU. The weighted loads for the two
CPUs would be 4.0 and 0.2 respectively and the load balancing code would
move 2 tasks resulting in one CPU with a load of 2.0 and the other with
load of 2.2. If this was considered to be a big enough imbalance to
justify moving a task and that task was moved using the current
move_tasks() then it would move the highest priority task that it found and
this would result in one CPU with a load of 3.0 and the other with a load
of 1.2 which would result in the movement of a task in the opposite
direction and so on -- infinite loop. If, on the other hand, an amount of
load to be moved is calculated from the imbalance (in this case 0.1) and
move_tasks() skips tasks until it find ones whose contributions to the
weighted load are less than this amount it would move two of the nice==19
tasks resulting in a system with 2 nice==0 and 2 nice=19 on each CPU with
loads of 2.1 for each CPU.
One of the advantages of this mechanism is that on a system where all tasks
have nice==0 the load balancing calculations would be mathematically
identical to the current load balancing code.
Notes:
struct task_struct:
has a new field load_weight which (in a trade off of space for speed)
stores the contribution that this task makes to a CPU's weighted load when
it is runnable.
struct runqueue:
has a new field raw_weighted_load which is the sum of the load_weight
values for the currently runnable tasks on this run queue. This field
always needs to be updated when nr_running is updated so two new inline
functions inc_nr_running() and dec_nr_running() have been created to make
sure that this happens. This also offers a convenient way to optimize away
this part of the smpnice mechanism when CONFIG_SMP is not defined.
int try_to_wake_up():
in this function the value SCHED_LOAD_BALANCE is used to represent the load
contribution of a single task in various calculations in the code that
decides which CPU to put the waking task on. While this would be a valid
on a system where the nice values for the runnable tasks were distributed
evenly around zero it will lead to anomalous load balancing if the
distribution is skewed in either direction. To overcome this problem
SCHED_LOAD_SCALE has been replaced by the load_weight for the relevant task
or by the average load_weight per task for the queue in question (as
appropriate).
int move_tasks():
The modifications to this function were complicated by the fact that
active_load_balance() uses it to move exactly one task without checking
whether an imbalance actually exists. This precluded the simple
overloading of max_nr_move with max_load_move and necessitated the addition
of the latter as an extra argument to the function. The internal
implementation is then modified to move up to max_nr_move tasks and
max_load_move of weighted load. This slightly complicates the code where
move_tasks() is called and if ever active_load_balance() is changed to not
use move_tasks() the implementation of move_tasks() should be simplified
accordingly.
struct sched_group *find_busiest_group():
Similar to try_to_wake_up(), there are places in this function where
SCHED_LOAD_SCALE is used to represent the load contribution of a single
task and the same issues are created. A similar solution is adopted except
that it is now the average per task contribution to a group's load (as
opposed to a run queue) that is required. As this value is not directly
available from the group it is calculated on the fly as the queues in the
groups are visited when determining the busiest group.
A key change to this function is that it is no longer to scale down
*imbalance on exit as move_tasks() uses the load in its scaled form.
void set_user_nice():
has been modified to update the task's load_weight field when it's nice
value and also to ensure that its run queue's raw_weighted_load field is
updated if it was runnable.
From: "Siddha, Suresh B" <suresh.b.siddha@intel.com>
With smpnice, sched groups with highest priority tasks can mask the imbalance
between the other sched groups with in the same domain. This patch fixes some
of the listed down scenarios by not considering the sched groups which are
lightly loaded.
a) on a simple 4-way MP system, if we have one high priority and 4 normal
priority tasks, with smpnice we would like to see the high priority task
scheduled on one cpu, two other cpus getting one normal task each and the
fourth cpu getting the remaining two normal tasks. but with current
smpnice extra normal priority task keeps jumping from one cpu to another
cpu having the normal priority task. This is because of the
busiest_has_loaded_cpus, nr_loaded_cpus logic.. We are not including the
cpu with high priority task in max_load calculations but including that in
total and avg_load calcuations.. leading to max_load < avg_load and load
balance between cpus running normal priority tasks(2 Vs 1) will always show
imbalanace as one normal priority and the extra normal priority task will
keep moving from one cpu to another cpu having normal priority task..
b) 4-way system with HT (8 logical processors). Package-P0 T0 has a
highest priority task, T1 is idle. Package-P1 Both T0 and T1 have 1 normal
priority task each.. P2 and P3 are idle. With this patch, one of the
normal priority tasks on P1 will be moved to P2 or P3..
c) With the current weighted smp nice calculations, it doesn't always make
sense to look at the highest weighted runqueue in the busy group..
Consider a load balance scenario on a DP with HT system, with Package-0
containing one high priority and one low priority, Package-1 containing one
low priority(with other thread being idle).. Package-1 thinks that it need
to take the low priority thread from Package-0. And find_busiest_queue()
returns the cpu thread with highest priority task.. And ultimately(with
help of active load balance) we move high priority task to Package-1. And
same continues with Package-0 now, moving high priority task from package-1
to package-0.. Even without the presence of active load balance, load
balance will fail to balance the above scenario.. Fix find_busiest_queue
to use "imbalance" when it is lightly loaded.
[kernel@kolivas.org: sched: store weighted load on up]
[kernel@kolivas.org: sched: add discrete weighted cpu load function]
[suresh.b.siddha@intel.com: sched: remove dead code]
Signed-off-by: Peter Williams <pwil3058@bigpond.com.au>
Cc: "Siddha, Suresh B" <suresh.b.siddha@intel.com>
Cc: "Chen, Kenneth W" <kenneth.w.chen@intel.com>
Acked-by: Ingo Molnar <mingo@elte.hu>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Con Kolivas <kernel@kolivas.org>
Cc: John Hawkes <hawkes@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-06-27 17:54:34 +08:00
|
|
|
#endif
|
2007-07-10 00:52:00 +08:00
|
|
|
|
2006-06-27 17:54:51 +08:00
|
|
|
int prio, static_prio, normal_prio;
|
2008-05-15 19:09:15 +08:00
|
|
|
unsigned int rt_priority;
|
2007-10-15 23:00:12 +08:00
|
|
|
const struct sched_class *sched_class;
|
2007-07-10 00:51:58 +08:00
|
|
|
struct sched_entity se;
|
2008-01-26 04:08:27 +08:00
|
|
|
struct sched_rt_entity rt;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-07-26 19:40:43 +08:00
|
|
|
#ifdef CONFIG_PREEMPT_NOTIFIERS
|
|
|
|
|
/* list of struct preempt_notifier: */
|
|
|
|
|
struct hlist_head preempt_notifiers;
|
|
|
|
|
#endif
|
|
|
|
|
|
2007-10-17 14:30:26 +08:00
|
|
|
/*
|
|
|
|
|
* fpu_counter contains the number of consecutive context switches
|
|
|
|
|
* that the FPU is used. If this is over a threshold, the lazy fpu
|
|
|
|
|
* saving becomes unlazy to save the trap. This is an unsigned char
|
|
|
|
|
* so that after 256 times the counter wraps and the behavior turns
|
|
|
|
|
* lazy again; this to deal with bursty apps that only use FPU for
|
|
|
|
|
* a short time
|
|
|
|
|
*/
|
|
|
|
|
unsigned char fpu_counter;
|
2006-09-29 16:59:40 +08:00
|
|
|
#ifdef CONFIG_BLK_DEV_IO_TRACE
|
2006-03-24 03:00:26 +08:00
|
|
|
unsigned int btrace_seq;
|
2006-09-29 16:59:40 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-05-08 15:23:41 +08:00
|
|
|
unsigned int policy;
|
2005-04-17 06:20:36 +08:00
|
|
|
cpumask_t cpus_allowed;
|
|
|
|
|
|
rcu: Merge preemptable-RCU functionality into hierarchical RCU
Create a kernel/rcutree_plugin.h file that contains definitions
for preemptable RCU (or, under the #else branch of the #ifdef,
empty definitions for the classic non-preemptable semantics).
These definitions fit into plugins defined in kernel/rcutree.c
for this purpose.
This variant of preemptable RCU uses a new algorithm whose
read-side expense is roughly that of classic hierarchical RCU
under CONFIG_PREEMPT. This new algorithm's update-side expense
is similar to that of classic hierarchical RCU, and, in absence
of read-side preemption or blocking, is exactly that of classic
hierarchical RCU. Perhaps more important, this new algorithm
has a much simpler implementation, saving well over 1,000 lines
of code compared to mainline's implementation of preemptable
RCU, which will hopefully be retired in favor of this new
algorithm.
The simplifications are obtained by maintaining per-task
nesting state for running tasks, and using a simple
lock-protected algorithm to handle accounting when tasks block
within RCU read-side critical sections, making use of lessons
learned while creating numerous user-level RCU implementations
over the past 18 months.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: akpm@linux-foundation.org
Cc: mathieu.desnoyers@polymtl.ca
Cc: josht@linux.vnet.ibm.com
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
LKML-Reference: <12509746134003-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-08-23 04:56:52 +08:00
|
|
|
#ifdef CONFIG_TREE_PREEMPT_RCU
|
2008-01-26 04:08:24 +08:00
|
|
|
int rcu_read_lock_nesting;
|
rcu: Merge preemptable-RCU functionality into hierarchical RCU
Create a kernel/rcutree_plugin.h file that contains definitions
for preemptable RCU (or, under the #else branch of the #ifdef,
empty definitions for the classic non-preemptable semantics).
These definitions fit into plugins defined in kernel/rcutree.c
for this purpose.
This variant of preemptable RCU uses a new algorithm whose
read-side expense is roughly that of classic hierarchical RCU
under CONFIG_PREEMPT. This new algorithm's update-side expense
is similar to that of classic hierarchical RCU, and, in absence
of read-side preemption or blocking, is exactly that of classic
hierarchical RCU. Perhaps more important, this new algorithm
has a much simpler implementation, saving well over 1,000 lines
of code compared to mainline's implementation of preemptable
RCU, which will hopefully be retired in favor of this new
algorithm.
The simplifications are obtained by maintaining per-task
nesting state for running tasks, and using a simple
lock-protected algorithm to handle accounting when tasks block
within RCU read-side critical sections, making use of lessons
learned while creating numerous user-level RCU implementations
over the past 18 months.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: akpm@linux-foundation.org
Cc: mathieu.desnoyers@polymtl.ca
Cc: josht@linux.vnet.ibm.com
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
LKML-Reference: <12509746134003-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-08-23 04:56:52 +08:00
|
|
|
char rcu_read_unlock_special;
|
2009-08-28 06:00:12 +08:00
|
|
|
struct rcu_node *rcu_blocked_node;
|
rcu: Merge preemptable-RCU functionality into hierarchical RCU
Create a kernel/rcutree_plugin.h file that contains definitions
for preemptable RCU (or, under the #else branch of the #ifdef,
empty definitions for the classic non-preemptable semantics).
These definitions fit into plugins defined in kernel/rcutree.c
for this purpose.
This variant of preemptable RCU uses a new algorithm whose
read-side expense is roughly that of classic hierarchical RCU
under CONFIG_PREEMPT. This new algorithm's update-side expense
is similar to that of classic hierarchical RCU, and, in absence
of read-side preemption or blocking, is exactly that of classic
hierarchical RCU. Perhaps more important, this new algorithm
has a much simpler implementation, saving well over 1,000 lines
of code compared to mainline's implementation of preemptable
RCU, which will hopefully be retired in favor of this new
algorithm.
The simplifications are obtained by maintaining per-task
nesting state for running tasks, and using a simple
lock-protected algorithm to handle accounting when tasks block
within RCU read-side critical sections, making use of lessons
learned while creating numerous user-level RCU implementations
over the past 18 months.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: akpm@linux-foundation.org
Cc: mathieu.desnoyers@polymtl.ca
Cc: josht@linux.vnet.ibm.com
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
LKML-Reference: <12509746134003-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-08-23 04:56:52 +08:00
|
|
|
struct list_head rcu_node_entry;
|
|
|
|
|
#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
|
2008-01-26 04:08:24 +08:00
|
|
|
|
2006-07-14 15:24:38 +08:00
|
|
|
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sched_info sched_info;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
struct list_head tasks;
|
sched: create "pushable_tasks" list to limit pushing to one attempt
The RT scheduler employs a "push/pull" design to actively balance tasks
within the system (on a per disjoint cpuset basis). When a task is
awoken, it is immediately determined if there are any lower priority
cpus which should be preempted. This is opposed to the way normal
SCHED_OTHER tasks behave, which will wait for a periodic rebalancing
operation to occur before spreading out load.
When a particular RQ has more than 1 active RT task, it is said to
be in an "overloaded" state. Once this occurs, the system enters
the active balancing mode, where it will try to push the task away,
or persuade a different cpu to pull it over. The system will stay
in this state until the system falls back below the <= 1 queued RT
task per RQ.
However, the current implementation suffers from a limitation in the
push logic. Once overloaded, all tasks (other than current) on the
RQ are analyzed on every push operation, even if it was previously
unpushable (due to affinity, etc). Whats more, the operation stops
at the first task that is unpushable and will not look at items
lower in the queue. This causes two problems:
1) We can have the same tasks analyzed over and over again during each
push, which extends out the fast path in the scheduler for no
gain. Consider a RQ that has dozens of tasks that are bound to a
core. Each one of those tasks will be encountered and skipped
for each push operation while they are queued.
2) There may be lower-priority tasks under the unpushable task that
could have been successfully pushed, but will never be considered
until either the unpushable task is cleared, or a pull operation
succeeds. The net result is a potential latency source for mid
priority tasks.
This patch aims to rectify these two conditions by introducing a new
priority sorted list: "pushable_tasks". A task is added to the list
each time a task is activated or preempted. It is removed from the
list any time it is deactivated, made current, or fails to push.
This works because a task only needs to be attempted to push once.
After an initial failure to push, the other cpus will eventually try to
pull the task when the conditions are proper. This also solves the
problem that we don't completely analyze all tasks due to encountering
an unpushable tasks. Now every task will have a push attempted (when
appropriate).
This reduces latency both by shorting the critical section of the
rq->lock for certain workloads, and by making sure the algorithm
considers all eligible tasks in the system.
[ rostedt: added a couple more BUG_ONs ]
Signed-off-by: Gregory Haskins <ghaskins@novell.com>
Acked-by: Steven Rostedt <srostedt@redhat.com>
2008-12-29 22:39:53 +08:00
|
|
|
struct plist_node pushable_tasks;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
struct mm_struct *mm, *active_mm;
|
2010-03-06 05:41:40 +08:00
|
|
|
#if defined(SPLIT_RSS_COUNTING)
|
|
|
|
|
struct task_rss_stat rss_stat;
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
/* task state */
|
2007-05-08 15:23:41 +08:00
|
|
|
int exit_state;
|
2005-04-17 06:20:36 +08:00
|
|
|
int exit_code, exit_signal;
|
|
|
|
|
int pdeath_signal; /* The signal sent when the parent dies */
|
|
|
|
|
/* ??? */
|
2007-05-08 15:23:41 +08:00
|
|
|
unsigned int personality;
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned did_exec:1;
|
2009-02-05 16:18:11 +08:00
|
|
|
unsigned in_execve:1; /* Tell the LSMs that the process is doing an
|
|
|
|
|
* execve */
|
2009-07-21 02:26:58 +08:00
|
|
|
unsigned in_iowait:1;
|
|
|
|
|
|
2009-06-15 23:17:47 +08:00
|
|
|
|
|
|
|
|
/* Revert to default priority/policy when forking */
|
|
|
|
|
unsigned sched_reset_on_fork:1;
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
pid_t pid;
|
|
|
|
|
pid_t tgid;
|
2006-09-26 16:52:38 +08:00
|
|
|
|
2009-08-18 14:06:02 +08:00
|
|
|
#ifdef CONFIG_CC_STACKPROTECTOR
|
2006-09-26 16:52:38 +08:00
|
|
|
/* Canary value for the -fstack-protector gcc feature */
|
|
|
|
|
unsigned long stack_canary;
|
2009-08-18 14:06:02 +08:00
|
|
|
#endif
|
2008-02-14 15:48:23 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* pointers to (original) parent process, youngest child, younger sibling,
|
|
|
|
|
* older sibling, respectively. (p->father can be replaced with
|
2008-03-25 09:36:23 +08:00
|
|
|
* p->real_parent->pid)
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2008-03-25 09:36:23 +08:00
|
|
|
struct task_struct *real_parent; /* real parent process */
|
|
|
|
|
struct task_struct *parent; /* recipient of SIGCHLD, wait4() reports */
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2008-03-25 09:36:23 +08:00
|
|
|
* children/sibling forms the list of my natural children
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
|
struct list_head children; /* list of my children */
|
|
|
|
|
struct list_head sibling; /* linkage in my parent's children list */
|
|
|
|
|
struct task_struct *group_leader; /* threadgroup leader */
|
|
|
|
|
|
2008-03-25 09:36:23 +08:00
|
|
|
/*
|
|
|
|
|
* ptraced is the list of tasks this task is using ptrace on.
|
|
|
|
|
* This includes both natural children and PTRACE_ATTACH targets.
|
|
|
|
|
* p->ptrace_entry is p's link on the p->parent->ptraced list.
|
|
|
|
|
*/
|
|
|
|
|
struct list_head ptraced;
|
|
|
|
|
struct list_head ptrace_entry;
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* PID/PID hash table linkage. */
|
[PATCH] pidhash: Refactor the pid hash table
Simplifies the code, reduces the need for 4 pid hash tables, and makes the
code more capable.
In the discussions I had with Oleg it was felt that to a large extent the
cleanup itself justified the work. With struct pid being dynamically
allocated meant we could create the hash table entry when the pid was
allocated and free the hash table entry when the pid was freed. Instead of
playing with the hash lists when ever a process would attach or detach to a
process.
For myself the fact that it gave what my previous task_ref patch gave for free
with simpler code was a big win. The problem is that if you hold a reference
to struct task_struct you lock in 10K of low memory. If you do that in a user
controllable way like /proc does, with an unprivileged but hostile user space
application with typical resource limits of 1000 fds and 100 processes I can
trigger the OOM killer by consuming all of low memory with task structs, on a
machine wight 1GB of low memory.
If I instead hold a reference to struct pid which holds a pointer to my
task_struct, I don't suffer from that problem because struct pid is 2 orders
of magnitude smaller. In fact struct pid is small enough that most other
kernel data structures dwarf it, so simply limiting the number of referring
data structures is enough to prevent exhaustion of low memory.
This splits the current struct pid into two structures, struct pid and struct
pid_link, and reduces our number of hash tables from PIDTYPE_MAX to just one.
struct pid_link is the per process linkage into the hash tables and lives in
struct task_struct. struct pid is given an indepedent lifetime, and holds
pointers to each of the pid types.
The independent life of struct pid simplifies attach_pid, and detach_pid,
because we are always manipulating the list of pids and not the hash table.
In addition in giving struct pid an indpendent life it makes the concept much
more powerful.
Kernel data structures can now embed a struct pid * instead of a pid_t and
not suffer from pid wrap around problems or from keeping unnecessarily
large amounts of memory allocated.
Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-31 18:31:42 +08:00
|
|
|
struct pid_link pids[PIDTYPE_MAX];
|
2006-03-29 08:11:25 +08:00
|
|
|
struct list_head thread_group;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
struct completion *vfork_done; /* for vfork() */
|
|
|
|
|
int __user *set_child_tid; /* CLONE_CHILD_SETTID */
|
|
|
|
|
int __user *clear_child_tid; /* CLONE_CHILD_CLEARTID */
|
|
|
|
|
|
2007-10-18 18:06:34 +08:00
|
|
|
cputime_t utime, stime, utimescaled, stimescaled;
|
2007-10-15 23:00:19 +08:00
|
|
|
cputime_t gtime;
|
2009-12-02 16:26:47 +08:00
|
|
|
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
|
2007-10-30 07:26:32 +08:00
|
|
|
cputime_t prev_utime, prev_stime;
|
2009-12-02 16:26:47 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned long nvcsw, nivcsw; /* context switch counts */
|
2007-07-16 14:39:42 +08:00
|
|
|
struct timespec start_time; /* monotonic time */
|
|
|
|
|
struct timespec real_start_time; /* boot based time */
|
2005-04-17 06:20:36 +08:00
|
|
|
/* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */
|
|
|
|
|
unsigned long min_flt, maj_flt;
|
|
|
|
|
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
struct task_cputime cputime_expires;
|
2005-04-17 06:20:36 +08:00
|
|
|
struct list_head cpu_timers[3];
|
|
|
|
|
|
|
|
|
|
/* process credentials */
|
2008-11-14 07:39:26 +08:00
|
|
|
const struct cred *real_cred; /* objective and real subjective task
|
|
|
|
|
* credentials (COW) */
|
|
|
|
|
const struct cred *cred; /* effective (overridable) subjective task
|
|
|
|
|
* credentials (COW) */
|
2009-05-08 20:55:22 +08:00
|
|
|
struct mutex cred_guard_mutex; /* guard against foreign influences on
|
|
|
|
|
* credential calculations
|
|
|
|
|
* (notably. ptrace) */
|
KEYS: Add a keyctl to install a process's session keyring on its parent [try #6]
Add a keyctl to install a process's session keyring onto its parent. This
replaces the parent's session keyring. Because the COW credential code does
not permit one process to change another process's credentials directly, the
change is deferred until userspace next starts executing again. Normally this
will be after a wait*() syscall.
To support this, three new security hooks have been provided:
cred_alloc_blank() to allocate unset security creds, cred_transfer() to fill in
the blank security creds and key_session_to_parent() - which asks the LSM if
the process may replace its parent's session keyring.
The replacement may only happen if the process has the same ownership details
as its parent, and the process has LINK permission on the session keyring, and
the session keyring is owned by the process, and the LSM permits it.
Note that this requires alteration to each architecture's notify_resume path.
This has been done for all arches barring blackfin, m68k* and xtensa, all of
which need assembly alteration to support TIF_NOTIFY_RESUME. This allows the
replacement to be performed at the point the parent process resumes userspace
execution.
This allows the userspace AFS pioctl emulation to fully emulate newpag() and
the VIOCSETTOK and VIOCSETTOK2 pioctls, all of which require the ability to
alter the parent process's PAG membership. However, since kAFS doesn't use
PAGs per se, but rather dumps the keys into the session keyring, the session
keyring of the parent must be replaced if, for example, VIOCSETTOK is passed
the newpag flag.
This can be tested with the following program:
#include <stdio.h>
#include <stdlib.h>
#include <keyutils.h>
#define KEYCTL_SESSION_TO_PARENT 18
#define OSERROR(X, S) do { if ((long)(X) == -1) { perror(S); exit(1); } } while(0)
int main(int argc, char **argv)
{
key_serial_t keyring, key;
long ret;
keyring = keyctl_join_session_keyring(argv[1]);
OSERROR(keyring, "keyctl_join_session_keyring");
key = add_key("user", "a", "b", 1, keyring);
OSERROR(key, "add_key");
ret = keyctl(KEYCTL_SESSION_TO_PARENT);
OSERROR(ret, "KEYCTL_SESSION_TO_PARENT");
return 0;
}
Compiled and linked with -lkeyutils, you should see something like:
[dhowells@andromeda ~]$ keyctl show
Session Keyring
-3 --alswrv 4043 4043 keyring: _ses
355907932 --alswrv 4043 -1 \_ keyring: _uid.4043
[dhowells@andromeda ~]$ /tmp/newpag
[dhowells@andromeda ~]$ keyctl show
Session Keyring
-3 --alswrv 4043 4043 keyring: _ses
1055658746 --alswrv 4043 4043 \_ user: a
[dhowells@andromeda ~]$ /tmp/newpag hello
[dhowells@andromeda ~]$ keyctl show
Session Keyring
-3 --alswrv 4043 4043 keyring: hello
340417692 --alswrv 4043 4043 \_ user: a
Where the test program creates a new session keyring, sticks a user key named
'a' into it and then installs it on its parent.
Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: James Morris <jmorris@namei.org>
2009-09-02 16:14:21 +08:00
|
|
|
struct cred *replacement_session_keyring; /* for KEYCTL_SESSION_TO_PARENT */
|
2008-11-14 07:39:16 +08:00
|
|
|
|
2005-05-06 07:16:12 +08:00
|
|
|
char comm[TASK_COMM_LEN]; /* executable name excluding path
|
|
|
|
|
- access with [gs]et_task_comm (which lock
|
|
|
|
|
it with task_lock())
|
Split 'flush_old_exec' into two functions
'flush_old_exec()' is the point of no return when doing an execve(), and
it is pretty badly misnamed. It doesn't just flush the old executable
environment, it also starts up the new one.
Which is very inconvenient for things like setting up the new
personality, because we want the new personality to affect the starting
of the new environment, but at the same time we do _not_ want the new
personality to take effect if flushing the old one fails.
As a result, the x86-64 '32-bit' personality is actually done using this
insane "I'm going to change the ABI, but I haven't done it yet" bit
(TIF_ABI_PENDING), with SET_PERSONALITY() not actually setting the
personality, but just the "pending" bit, so that "flush_thread()" can do
the actual personality magic.
This patch in no way changes any of that insanity, but it does split the
'flush_old_exec()' function up into a preparatory part that can fail
(still called flush_old_exec()), and a new part that will actually set
up the new exec environment (setup_new_exec()). All callers are changed
to trivially comply with the new world order.
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
Cc: stable@kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-01-29 14:14:42 +08:00
|
|
|
- initialized normally by setup_new_exec */
|
2005-04-17 06:20:36 +08:00
|
|
|
/* file system info */
|
|
|
|
|
int link_count, total_link_count;
|
2006-09-29 16:59:40 +08:00
|
|
|
#ifdef CONFIG_SYSVIPC
|
2005-04-17 06:20:36 +08:00
|
|
|
/* ipc stuff */
|
|
|
|
|
struct sysv_sem sysvsem;
|
2006-09-29 16:59:40 +08:00
|
|
|
#endif
|
2009-01-16 03:08:40 +08:00
|
|
|
#ifdef CONFIG_DETECT_HUNG_TASK
|
2008-01-26 04:08:02 +08:00
|
|
|
/* hung task detection */
|
|
|
|
|
unsigned long last_switch_count;
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
/* CPU-specific state of this task */
|
|
|
|
|
struct thread_struct thread;
|
|
|
|
|
/* filesystem information */
|
|
|
|
|
struct fs_struct *fs;
|
|
|
|
|
/* open file information */
|
|
|
|
|
struct files_struct *files;
|
2006-10-02 17:18:08 +08:00
|
|
|
/* namespaces */
|
2006-10-02 17:18:06 +08:00
|
|
|
struct nsproxy *nsproxy;
|
2005-04-17 06:20:36 +08:00
|
|
|
/* signal handlers */
|
|
|
|
|
struct signal_struct *signal;
|
|
|
|
|
struct sighand_struct *sighand;
|
|
|
|
|
|
|
|
|
|
sigset_t blocked, real_blocked;
|
2008-04-30 15:53:09 +08:00
|
|
|
sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */
|
2005-04-17 06:20:36 +08:00
|
|
|
struct sigpending pending;
|
|
|
|
|
|
|
|
|
|
unsigned long sas_ss_sp;
|
|
|
|
|
size_t sas_ss_size;
|
|
|
|
|
int (*notifier)(void *priv);
|
|
|
|
|
void *notifier_data;
|
|
|
|
|
sigset_t *notifier_mask;
|
|
|
|
|
struct audit_context *audit_context;
|
2008-01-10 17:53:18 +08:00
|
|
|
#ifdef CONFIG_AUDITSYSCALL
|
|
|
|
|
uid_t loginuid;
|
2008-01-08 23:06:53 +08:00
|
|
|
unsigned int sessionid;
|
2008-01-10 17:53:18 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
seccomp_t seccomp;
|
|
|
|
|
|
|
|
|
|
/* Thread group tracking */
|
|
|
|
|
u32 parent_exec_id;
|
|
|
|
|
u32 self_exec_id;
|
cpuset,mm: update tasks' mems_allowed in time
Fix allocating page cache/slab object on the unallowed node when memory
spread is set by updating tasks' mems_allowed after its cpuset's mems is
changed.
In order to update tasks' mems_allowed in time, we must modify the code of
memory policy. Because the memory policy is applied in the process's
context originally. After applying this patch, one task directly
manipulates anothers mems_allowed, and we use alloc_lock in the
task_struct to protect mems_allowed and memory policy of the task.
But in the fast path, we didn't use lock to protect them, because adding a
lock may lead to performance regression. But if we don't add a lock,the
task might see no nodes when changing cpuset's mems_allowed to some
non-overlapping set. In order to avoid it, we set all new allowed nodes,
then clear newly disallowed ones.
[lee.schermerhorn@hp.com:
The rework of mpol_new() to extract the adjusting of the node mask to
apply cpuset and mpol flags "context" breaks set_mempolicy() and mbind()
with MPOL_PREFERRED and a NULL nodemask--i.e., explicit local
allocation. Fix this by adding the check for MPOL_PREFERRED and empty
node mask to mpol_new_mpolicy().
Remove the now unneeded 'nodes = NULL' from mpol_new().
Note that mpol_new_mempolicy() is always called with a non-NULL
'nodes' parameter now that it has been removed from mpol_new().
Therefore, we don't need to test nodes for NULL before testing it for
'empty'. However, just to be extra paranoid, add a VM_BUG_ON() to
verify this assumption.]
[lee.schermerhorn@hp.com:
I don't think the function name 'mpol_new_mempolicy' is descriptive
enough to differentiate it from mpol_new().
This function applies cpuset set context, usually constraining nodes
to those allowed by the cpuset. However, when the 'RELATIVE_NODES flag
is set, it also translates the nodes. So I settled on
'mpol_set_nodemask()', because the comment block for mpol_new() mentions
that we need to call this function to "set nodes".
Some additional minor line length, whitespace and typo cleanup.]
Signed-off-by: Miao Xie <miaox@cn.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Paul Menage <menage@google.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-06-17 06:31:49 +08:00
|
|
|
/* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed,
|
|
|
|
|
* mempolicy */
|
2005-04-17 06:20:36 +08:00
|
|
|
spinlock_t alloc_lock;
|
|
|
|
|
|
2009-03-24 01:28:15 +08:00
|
|
|
#ifdef CONFIG_GENERIC_HARDIRQS
|
|
|
|
|
/* IRQ handler threads */
|
|
|
|
|
struct irqaction *irqaction;
|
|
|
|
|
#endif
|
|
|
|
|
|
2006-06-27 17:54:51 +08:00
|
|
|
/* Protection of the PI data structures: */
|
2009-11-17 21:54:03 +08:00
|
|
|
raw_spinlock_t pi_lock;
|
2006-06-27 17:54:51 +08:00
|
|
|
|
2006-06-27 17:54:53 +08:00
|
|
|
#ifdef CONFIG_RT_MUTEXES
|
|
|
|
|
/* PI waiters blocked on a rt_mutex held by this task */
|
|
|
|
|
struct plist_head pi_waiters;
|
|
|
|
|
/* Deadlock detection and priority inheritance handling */
|
|
|
|
|
struct rt_mutex_waiter *pi_blocked_on;
|
|
|
|
|
#endif
|
|
|
|
|
|
2006-01-10 07:59:20 +08:00
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
|
/* mutex deadlock detection */
|
|
|
|
|
struct mutex_waiter *blocked_on;
|
|
|
|
|
#endif
|
2006-07-03 15:24:42 +08:00
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
|
|
|
unsigned int irq_events;
|
|
|
|
|
unsigned long hardirq_enable_ip;
|
|
|
|
|
unsigned long hardirq_disable_ip;
|
2009-11-30 13:59:44 +08:00
|
|
|
unsigned int hardirq_enable_event;
|
2006-07-03 15:24:42 +08:00
|
|
|
unsigned int hardirq_disable_event;
|
2009-11-30 13:59:44 +08:00
|
|
|
int hardirqs_enabled;
|
|
|
|
|
int hardirq_context;
|
2006-07-03 15:24:42 +08:00
|
|
|
unsigned long softirq_disable_ip;
|
|
|
|
|
unsigned long softirq_enable_ip;
|
2009-11-30 13:59:44 +08:00
|
|
|
unsigned int softirq_disable_event;
|
2006-07-03 15:24:42 +08:00
|
|
|
unsigned int softirq_enable_event;
|
2009-11-30 13:59:44 +08:00
|
|
|
int softirqs_enabled;
|
2006-07-03 15:24:42 +08:00
|
|
|
int softirq_context;
|
|
|
|
|
#endif
|
[PATCH] lockdep: core
Do 'make oldconfig' and accept all the defaults for new config options -
reboot into the kernel and if everything goes well it should boot up fine and
you should have /proc/lockdep and /proc/lockdep_stats files.
Typically if the lock validator finds some problem it will print out
voluminous debug output that begins with "BUG: ..." and which syslog output
can be used by kernel developers to figure out the precise locking scenario.
What does the lock validator do? It "observes" and maps all locking rules as
they occur dynamically (as triggered by the kernel's natural use of spinlocks,
rwlocks, mutexes and rwsems). Whenever the lock validator subsystem detects a
new locking scenario, it validates this new rule against the existing set of
rules. If this new rule is consistent with the existing set of rules then the
new rule is added transparently and the kernel continues as normal. If the
new rule could create a deadlock scenario then this condition is printed out.
When determining validity of locking, all possible "deadlock scenarios" are
considered: assuming arbitrary number of CPUs, arbitrary irq context and task
context constellations, running arbitrary combinations of all the existing
locking scenarios. In a typical system this means millions of separate
scenarios. This is why we call it a "locking correctness" validator - for all
rules that are observed the lock validator proves it with mathematical
certainty that a deadlock could not occur (assuming that the lock validator
implementation itself is correct and its internal data structures are not
corrupted by some other kernel subsystem). [see more details and conditionals
of this statement in include/linux/lockdep.h and
Documentation/lockdep-design.txt]
Furthermore, this "all possible scenarios" property of the validator also
enables the finding of complex, highly unlikely multi-CPU multi-context races
via single single-context rules, increasing the likelyhood of finding bugs
drastically. In practical terms: the lock validator already found a bug in
the upstream kernel that could only occur on systems with 3 or more CPUs, and
which needed 3 very unlikely code sequences to occur at once on the 3 CPUs.
That bug was found and reported on a single-CPU system (!). So in essence a
race will be found "piecemail-wise", triggering all the necessary components
for the race, without having to reproduce the race scenario itself! In its
short existence the lock validator found and reported many bugs before they
actually caused a real deadlock.
To further increase the efficiency of the validator, the mapping is not per
"lock instance", but per "lock-class". For example, all struct inode objects
in the kernel have inode->inotify_mutex. If there are 10,000 inodes cached,
then there are 10,000 lock objects. But ->inotify_mutex is a single "lock
type", and all locking activities that occur against ->inotify_mutex are
"unified" into this single lock-class. The advantage of the lock-class
approach is that all historical ->inotify_mutex uses are mapped into a single
(and as narrow as possible) set of locking rules - regardless of how many
different tasks or inode structures it took to build this set of rules. The
set of rules persist during the lifetime of the kernel.
To see the rough magnitude of checking that the lock validator does, here's a
portion of /proc/lockdep_stats, fresh after bootup:
lock-classes: 694 [max: 2048]
direct dependencies: 1598 [max: 8192]
indirect dependencies: 17896
all direct dependencies: 16206
dependency chains: 1910 [max: 8192]
in-hardirq chains: 17
in-softirq chains: 105
in-process chains: 1065
stack-trace entries: 38761 [max: 131072]
combined max dependencies: 2033928
hardirq-safe locks: 24
hardirq-unsafe locks: 176
softirq-safe locks: 53
softirq-unsafe locks: 137
irq-safe locks: 59
irq-unsafe locks: 176
The lock validator has observed 1598 actual single-thread locking patterns,
and has validated all possible 2033928 distinct locking scenarios.
More details about the design of the lock validator can be found in
Documentation/lockdep-design.txt, which can also found at:
http://redhat.com/~mingo/lockdep-patches/lockdep-design.txt
[bunk@stusta.de: cleanups]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Arjan van de Ven <arjan@linux.intel.com>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-07-03 15:24:50 +08:00
|
|
|
#ifdef CONFIG_LOCKDEP
|
2008-02-26 06:02:48 +08:00
|
|
|
# define MAX_LOCK_DEPTH 48UL
|
[PATCH] lockdep: core
Do 'make oldconfig' and accept all the defaults for new config options -
reboot into the kernel and if everything goes well it should boot up fine and
you should have /proc/lockdep and /proc/lockdep_stats files.
Typically if the lock validator finds some problem it will print out
voluminous debug output that begins with "BUG: ..." and which syslog output
can be used by kernel developers to figure out the precise locking scenario.
What does the lock validator do? It "observes" and maps all locking rules as
they occur dynamically (as triggered by the kernel's natural use of spinlocks,
rwlocks, mutexes and rwsems). Whenever the lock validator subsystem detects a
new locking scenario, it validates this new rule against the existing set of
rules. If this new rule is consistent with the existing set of rules then the
new rule is added transparently and the kernel continues as normal. If the
new rule could create a deadlock scenario then this condition is printed out.
When determining validity of locking, all possible "deadlock scenarios" are
considered: assuming arbitrary number of CPUs, arbitrary irq context and task
context constellations, running arbitrary combinations of all the existing
locking scenarios. In a typical system this means millions of separate
scenarios. This is why we call it a "locking correctness" validator - for all
rules that are observed the lock validator proves it with mathematical
certainty that a deadlock could not occur (assuming that the lock validator
implementation itself is correct and its internal data structures are not
corrupted by some other kernel subsystem). [see more details and conditionals
of this statement in include/linux/lockdep.h and
Documentation/lockdep-design.txt]
Furthermore, this "all possible scenarios" property of the validator also
enables the finding of complex, highly unlikely multi-CPU multi-context races
via single single-context rules, increasing the likelyhood of finding bugs
drastically. In practical terms: the lock validator already found a bug in
the upstream kernel that could only occur on systems with 3 or more CPUs, and
which needed 3 very unlikely code sequences to occur at once on the 3 CPUs.
That bug was found and reported on a single-CPU system (!). So in essence a
race will be found "piecemail-wise", triggering all the necessary components
for the race, without having to reproduce the race scenario itself! In its
short existence the lock validator found and reported many bugs before they
actually caused a real deadlock.
To further increase the efficiency of the validator, the mapping is not per
"lock instance", but per "lock-class". For example, all struct inode objects
in the kernel have inode->inotify_mutex. If there are 10,000 inodes cached,
then there are 10,000 lock objects. But ->inotify_mutex is a single "lock
type", and all locking activities that occur against ->inotify_mutex are
"unified" into this single lock-class. The advantage of the lock-class
approach is that all historical ->inotify_mutex uses are mapped into a single
(and as narrow as possible) set of locking rules - regardless of how many
different tasks or inode structures it took to build this set of rules. The
set of rules persist during the lifetime of the kernel.
To see the rough magnitude of checking that the lock validator does, here's a
portion of /proc/lockdep_stats, fresh after bootup:
lock-classes: 694 [max: 2048]
direct dependencies: 1598 [max: 8192]
indirect dependencies: 17896
all direct dependencies: 16206
dependency chains: 1910 [max: 8192]
in-hardirq chains: 17
in-softirq chains: 105
in-process chains: 1065
stack-trace entries: 38761 [max: 131072]
combined max dependencies: 2033928
hardirq-safe locks: 24
hardirq-unsafe locks: 176
softirq-safe locks: 53
softirq-unsafe locks: 137
irq-safe locks: 59
irq-unsafe locks: 176
The lock validator has observed 1598 actual single-thread locking patterns,
and has validated all possible 2033928 distinct locking scenarios.
More details about the design of the lock validator can be found in
Documentation/lockdep-design.txt, which can also found at:
http://redhat.com/~mingo/lockdep-patches/lockdep-design.txt
[bunk@stusta.de: cleanups]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Arjan van de Ven <arjan@linux.intel.com>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-07-03 15:24:50 +08:00
|
|
|
u64 curr_chain_key;
|
|
|
|
|
int lockdep_depth;
|
|
|
|
|
unsigned int lockdep_recursion;
|
2008-05-15 19:09:15 +08:00
|
|
|
struct held_lock held_locks[MAX_LOCK_DEPTH];
|
lockdep: annotate reclaim context (__GFP_NOFS)
Here is another version, with the incremental patch rolled up, and
added reclaim context annotation to kswapd, and allocation tracing
to slab allocators (which may only ever reach the page allocator
in rare cases, so it is good to put annotations here too).
Haven't tested this version as such, but it should be getting closer
to merge worthy ;)
--
After noticing some code in mm/filemap.c accidentally perform a __GFP_FS
allocation when it should not have been, I thought it might be a good idea to
try to catch this kind of thing with lockdep.
I coded up a little idea that seems to work. Unfortunately the system has to
actually be in __GFP_FS page reclaim, then take the lock, before it will mark
it. But at least that might still be some orders of magnitude more common
(and more debuggable) than an actual deadlock condition, so we have some
improvement I hope (the concept is no less complete than discovery of a lock's
interrupt contexts).
I guess we could even do the same thing with __GFP_IO (normal reclaim), and
even GFP_NOIO locks too... but filesystems will have the most locks and fiddly
code paths, so let's start there and see how it goes.
It *seems* to work. I did a quick test.
=================================
[ INFO: inconsistent lock state ]
2.6.28-rc6-00007-ged31348-dirty #26
---------------------------------
inconsistent {in-reclaim-W} -> {ov-reclaim-W} usage.
modprobe/8526 [HC0[0]:SC0[0]:HE1:SE1] takes:
(testlock){--..}, at: [<ffffffffa0020055>] brd_init+0x55/0x216 [brd]
{in-reclaim-W} state was registered at:
[<ffffffff80267bdb>] __lock_acquire+0x75b/0x1a60
[<ffffffff80268f71>] lock_acquire+0x91/0xc0
[<ffffffff8070f0e1>] mutex_lock_nested+0xb1/0x310
[<ffffffffa002002b>] brd_init+0x2b/0x216 [brd]
[<ffffffff8020903b>] _stext+0x3b/0x170
[<ffffffff80272ebf>] sys_init_module+0xaf/0x1e0
[<ffffffff8020c3fb>] system_call_fastpath+0x16/0x1b
[<ffffffffffffffff>] 0xffffffffffffffff
irq event stamp: 3929
hardirqs last enabled at (3929): [<ffffffff8070f2b5>] mutex_lock_nested+0x285/0x310
hardirqs last disabled at (3928): [<ffffffff8070f089>] mutex_lock_nested+0x59/0x310
softirqs last enabled at (3732): [<ffffffff8061f623>] sk_filter+0x83/0xe0
softirqs last disabled at (3730): [<ffffffff8061f5b6>] sk_filter+0x16/0xe0
other info that might help us debug this:
1 lock held by modprobe/8526:
#0: (testlock){--..}, at: [<ffffffffa0020055>] brd_init+0x55/0x216 [brd]
stack backtrace:
Pid: 8526, comm: modprobe Not tainted 2.6.28-rc6-00007-ged31348-dirty #26
Call Trace:
[<ffffffff80265483>] print_usage_bug+0x193/0x1d0
[<ffffffff80266530>] mark_lock+0xaf0/0xca0
[<ffffffff80266735>] mark_held_locks+0x55/0xc0
[<ffffffffa0020000>] ? brd_init+0x0/0x216 [brd]
[<ffffffff802667ca>] trace_reclaim_fs+0x2a/0x60
[<ffffffff80285005>] __alloc_pages_internal+0x475/0x580
[<ffffffff8070f29e>] ? mutex_lock_nested+0x26e/0x310
[<ffffffffa0020000>] ? brd_init+0x0/0x216 [brd]
[<ffffffffa002006a>] brd_init+0x6a/0x216 [brd]
[<ffffffffa0020000>] ? brd_init+0x0/0x216 [brd]
[<ffffffff8020903b>] _stext+0x3b/0x170
[<ffffffff8070f8b9>] ? mutex_unlock+0x9/0x10
[<ffffffff8070f83d>] ? __mutex_unlock_slowpath+0x10d/0x180
[<ffffffff802669ec>] ? trace_hardirqs_on_caller+0x12c/0x190
[<ffffffff80272ebf>] sys_init_module+0xaf/0x1e0
[<ffffffff8020c3fb>] system_call_fastpath+0x16/0x1b
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-01-21 15:12:39 +08:00
|
|
|
gfp_t lockdep_reclaim_gfp;
|
[PATCH] lockdep: core
Do 'make oldconfig' and accept all the defaults for new config options -
reboot into the kernel and if everything goes well it should boot up fine and
you should have /proc/lockdep and /proc/lockdep_stats files.
Typically if the lock validator finds some problem it will print out
voluminous debug output that begins with "BUG: ..." and which syslog output
can be used by kernel developers to figure out the precise locking scenario.
What does the lock validator do? It "observes" and maps all locking rules as
they occur dynamically (as triggered by the kernel's natural use of spinlocks,
rwlocks, mutexes and rwsems). Whenever the lock validator subsystem detects a
new locking scenario, it validates this new rule against the existing set of
rules. If this new rule is consistent with the existing set of rules then the
new rule is added transparently and the kernel continues as normal. If the
new rule could create a deadlock scenario then this condition is printed out.
When determining validity of locking, all possible "deadlock scenarios" are
considered: assuming arbitrary number of CPUs, arbitrary irq context and task
context constellations, running arbitrary combinations of all the existing
locking scenarios. In a typical system this means millions of separate
scenarios. This is why we call it a "locking correctness" validator - for all
rules that are observed the lock validator proves it with mathematical
certainty that a deadlock could not occur (assuming that the lock validator
implementation itself is correct and its internal data structures are not
corrupted by some other kernel subsystem). [see more details and conditionals
of this statement in include/linux/lockdep.h and
Documentation/lockdep-design.txt]
Furthermore, this "all possible scenarios" property of the validator also
enables the finding of complex, highly unlikely multi-CPU multi-context races
via single single-context rules, increasing the likelyhood of finding bugs
drastically. In practical terms: the lock validator already found a bug in
the upstream kernel that could only occur on systems with 3 or more CPUs, and
which needed 3 very unlikely code sequences to occur at once on the 3 CPUs.
That bug was found and reported on a single-CPU system (!). So in essence a
race will be found "piecemail-wise", triggering all the necessary components
for the race, without having to reproduce the race scenario itself! In its
short existence the lock validator found and reported many bugs before they
actually caused a real deadlock.
To further increase the efficiency of the validator, the mapping is not per
"lock instance", but per "lock-class". For example, all struct inode objects
in the kernel have inode->inotify_mutex. If there are 10,000 inodes cached,
then there are 10,000 lock objects. But ->inotify_mutex is a single "lock
type", and all locking activities that occur against ->inotify_mutex are
"unified" into this single lock-class. The advantage of the lock-class
approach is that all historical ->inotify_mutex uses are mapped into a single
(and as narrow as possible) set of locking rules - regardless of how many
different tasks or inode structures it took to build this set of rules. The
set of rules persist during the lifetime of the kernel.
To see the rough magnitude of checking that the lock validator does, here's a
portion of /proc/lockdep_stats, fresh after bootup:
lock-classes: 694 [max: 2048]
direct dependencies: 1598 [max: 8192]
indirect dependencies: 17896
all direct dependencies: 16206
dependency chains: 1910 [max: 8192]
in-hardirq chains: 17
in-softirq chains: 105
in-process chains: 1065
stack-trace entries: 38761 [max: 131072]
combined max dependencies: 2033928
hardirq-safe locks: 24
hardirq-unsafe locks: 176
softirq-safe locks: 53
softirq-unsafe locks: 137
irq-safe locks: 59
irq-unsafe locks: 176
The lock validator has observed 1598 actual single-thread locking patterns,
and has validated all possible 2033928 distinct locking scenarios.
More details about the design of the lock validator can be found in
Documentation/lockdep-design.txt, which can also found at:
http://redhat.com/~mingo/lockdep-patches/lockdep-design.txt
[bunk@stusta.de: cleanups]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Arjan van de Ven <arjan@linux.intel.com>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-07-03 15:24:50 +08:00
|
|
|
#endif
|
2006-01-10 07:59:20 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* journalling filesystem info */
|
|
|
|
|
void *journal_info;
|
|
|
|
|
|
When stacked block devices are in-use (e.g. md or dm), the recursive calls
to generic_make_request can use up a lot of space, and we would rather they
didn't.
As generic_make_request is a void function, and as it is generally not
expected that it will have any effect immediately, it is safe to delay any
call to generic_make_request until there is sufficient stack space
available.
As ->bi_next is reserved for the driver to use, it can have no valid value
when generic_make_request is called, and as __make_request implicitly
assumes it will be NULL (ELEVATOR_BACK_MERGE fork of switch) we can be
certain that all callers set it to NULL. We can therefore safely use
bi_next to link pending requests together, providing we clear it before
making the real call.
So, we choose to allow each thread to only be active in one
generic_make_request at a time. If a subsequent (recursive) call is made,
the bio is linked into a per-thread list, and is handled when the active
call completes.
As the list of pending bios is per-thread, there are no locking issues to
worry about.
I say above that it is "safe to delay any call...". There are, however,
some behaviours of a make_request_fn which would make it unsafe. These
include any behaviour that assumes anything will have changed after a
recursive call to generic_make_request.
These could include:
- waiting for that call to finish and call it's bi_end_io function.
md use to sometimes do this (marking the superblock dirty before
completing a write) but doesn't any more
- inspecting the bio for fields that generic_make_request might
change, such as bi_sector or bi_bdev. It is hard to see a good
reason for this, and I don't think anyone actually does it.
- inspecing the queue to see if, e.g. it is 'full' yet. Again, I
think this is very unlikely to be useful, or to be done.
Signed-off-by: Neil Brown <neilb@suse.de>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: <dm-devel@redhat.com>
Alasdair G Kergon <agk@redhat.com> said:
I can see nothing wrong with this in principle.
For device-mapper at the moment though it's essential that, while the bio
mappings may now get delayed, they still get processed in exactly
the same order as they were passed to generic_make_request().
My main concern is whether the timing changes implicit in this patch
will make the rare data-corrupting races in the existing snapshot code
more likely. (I'm working on a fix for these races, but the unfinished
patch is already several hundred lines long.)
It would be helpful if some people on this mailing list would test
this patch in various scenarios and report back.
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
2007-05-01 15:53:42 +08:00
|
|
|
/* stacked block device info */
|
2010-02-23 15:55:42 +08:00
|
|
|
struct bio_list *bio_list;
|
When stacked block devices are in-use (e.g. md or dm), the recursive calls
to generic_make_request can use up a lot of space, and we would rather they
didn't.
As generic_make_request is a void function, and as it is generally not
expected that it will have any effect immediately, it is safe to delay any
call to generic_make_request until there is sufficient stack space
available.
As ->bi_next is reserved for the driver to use, it can have no valid value
when generic_make_request is called, and as __make_request implicitly
assumes it will be NULL (ELEVATOR_BACK_MERGE fork of switch) we can be
certain that all callers set it to NULL. We can therefore safely use
bi_next to link pending requests together, providing we clear it before
making the real call.
So, we choose to allow each thread to only be active in one
generic_make_request at a time. If a subsequent (recursive) call is made,
the bio is linked into a per-thread list, and is handled when the active
call completes.
As the list of pending bios is per-thread, there are no locking issues to
worry about.
I say above that it is "safe to delay any call...". There are, however,
some behaviours of a make_request_fn which would make it unsafe. These
include any behaviour that assumes anything will have changed after a
recursive call to generic_make_request.
These could include:
- waiting for that call to finish and call it's bi_end_io function.
md use to sometimes do this (marking the superblock dirty before
completing a write) but doesn't any more
- inspecting the bio for fields that generic_make_request might
change, such as bi_sector or bi_bdev. It is hard to see a good
reason for this, and I don't think anyone actually does it.
- inspecing the queue to see if, e.g. it is 'full' yet. Again, I
think this is very unlikely to be useful, or to be done.
Signed-off-by: Neil Brown <neilb@suse.de>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: <dm-devel@redhat.com>
Alasdair G Kergon <agk@redhat.com> said:
I can see nothing wrong with this in principle.
For device-mapper at the moment though it's essential that, while the bio
mappings may now get delayed, they still get processed in exactly
the same order as they were passed to generic_make_request().
My main concern is whether the timing changes implicit in this patch
will make the rare data-corrupting races in the existing snapshot code
more likely. (I'm working on a fix for these races, but the unfinished
patch is already several hundred lines long.)
It would be helpful if some people on this mailing list would test
this patch in various scenarios and report back.
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
2007-05-01 15:53:42 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* VM state */
|
|
|
|
|
struct reclaim_state *reclaim_state;
|
|
|
|
|
|
|
|
|
|
struct backing_dev_info *backing_dev_info;
|
|
|
|
|
|
|
|
|
|
struct io_context *io_context;
|
|
|
|
|
|
|
|
|
|
unsigned long ptrace_message;
|
|
|
|
|
siginfo_t *last_siginfo; /* For ptrace use. */
|
2006-12-10 18:19:19 +08:00
|
|
|
struct task_io_accounting ioac;
|
2006-10-01 14:28:59 +08:00
|
|
|
#if defined(CONFIG_TASK_XACCT)
|
2005-04-17 06:20:36 +08:00
|
|
|
u64 acct_rss_mem1; /* accumulated rss usage */
|
|
|
|
|
u64 acct_vm_mem1; /* accumulated virtual memory usage */
|
2008-07-25 16:48:40 +08:00
|
|
|
cputime_t acct_timexpd; /* stime + utime since last update */
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
|
#ifdef CONFIG_CPUSETS
|
cpuset,mm: update tasks' mems_allowed in time
Fix allocating page cache/slab object on the unallowed node when memory
spread is set by updating tasks' mems_allowed after its cpuset's mems is
changed.
In order to update tasks' mems_allowed in time, we must modify the code of
memory policy. Because the memory policy is applied in the process's
context originally. After applying this patch, one task directly
manipulates anothers mems_allowed, and we use alloc_lock in the
task_struct to protect mems_allowed and memory policy of the task.
But in the fast path, we didn't use lock to protect them, because adding a
lock may lead to performance regression. But if we don't add a lock,the
task might see no nodes when changing cpuset's mems_allowed to some
non-overlapping set. In order to avoid it, we set all new allowed nodes,
then clear newly disallowed ones.
[lee.schermerhorn@hp.com:
The rework of mpol_new() to extract the adjusting of the node mask to
apply cpuset and mpol flags "context" breaks set_mempolicy() and mbind()
with MPOL_PREFERRED and a NULL nodemask--i.e., explicit local
allocation. Fix this by adding the check for MPOL_PREFERRED and empty
node mask to mpol_new_mpolicy().
Remove the now unneeded 'nodes = NULL' from mpol_new().
Note that mpol_new_mempolicy() is always called with a non-NULL
'nodes' parameter now that it has been removed from mpol_new().
Therefore, we don't need to test nodes for NULL before testing it for
'empty'. However, just to be extra paranoid, add a VM_BUG_ON() to
verify this assumption.]
[lee.schermerhorn@hp.com:
I don't think the function name 'mpol_new_mempolicy' is descriptive
enough to differentiate it from mpol_new().
This function applies cpuset set context, usually constraining nodes
to those allowed by the cpuset. However, when the 'RELATIVE_NODES flag
is set, it also translates the nodes. So I settled on
'mpol_set_nodemask()', because the comment block for mpol_new() mentions
that we need to call this function to "set nodes".
Some additional minor line length, whitespace and typo cleanup.]
Signed-off-by: Miao Xie <miaox@cn.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Paul Menage <menage@google.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-06-17 06:31:49 +08:00
|
|
|
nodemask_t mems_allowed; /* Protected by alloc_lock */
|
[PATCH] cpuset memory spread basic implementation
This patch provides the implementation and cpuset interface for an alternative
memory allocation policy that can be applied to certain kinds of memory
allocations, such as the page cache (file system buffers) and some slab caches
(such as inode caches).
The policy is called "memory spreading." If enabled, it spreads out these
kinds of memory allocations over all the nodes allowed to a task, instead of
preferring to place them on the node where the task is executing.
All other kinds of allocations, including anonymous pages for a tasks stack
and data regions, are not affected by this policy choice, and continue to be
allocated preferring the node local to execution, as modified by the NUMA
mempolicy.
There are two boolean flag files per cpuset that control where the kernel
allocates pages for the file system buffers and related in kernel data
structures. They are called 'memory_spread_page' and 'memory_spread_slab'.
If the per-cpuset boolean flag file 'memory_spread_page' is set, then the
kernel will spread the file system buffers (page cache) evenly over all the
nodes that the faulting task is allowed to use, instead of preferring to put
those pages on the node where the task is running.
If the per-cpuset boolean flag file 'memory_spread_slab' is set, then the
kernel will spread some file system related slab caches, such as for inodes
and dentries evenly over all the nodes that the faulting task is allowed to
use, instead of preferring to put those pages on the node where the task is
running.
The implementation is simple. Setting the cpuset flags 'memory_spread_page'
or 'memory_spread_cache' turns on the per-process flags PF_SPREAD_PAGE or
PF_SPREAD_SLAB, respectively, for each task that is in the cpuset or
subsequently joins that cpuset. In subsequent patches, the page allocation
calls for the affected page cache and slab caches are modified to perform an
inline check for these flags, and if set, a call to a new routine
cpuset_mem_spread_node() returns the node to prefer for the allocation.
The cpuset_mem_spread_node() routine is also simple. It uses the value of a
per-task rotor cpuset_mem_spread_rotor to select the next node in the current
tasks mems_allowed to prefer for the allocation.
This policy can provide substantial improvements for jobs that need to place
thread local data on the corresponding node, but that need to access large
file system data sets that need to be spread across the several nodes in the
jobs cpuset in order to fit. Without this patch, especially for jobs that
might have one thread reading in the data set, the memory allocation across
the nodes in the jobs cpuset can become very uneven.
A couple of Copyright year ranges are updated as well. And a couple of email
addresses that can be found in the MAINTAINERS file are removed.
Signed-off-by: Paul Jackson <pj@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-24 19:16:03 +08:00
|
|
|
int cpuset_mem_spread_rotor;
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
Task Control Groups: basic task cgroup framework
Generic Process Control Groups
--------------------------
There have recently been various proposals floating around for
resource management/accounting and other task grouping subsystems in
the kernel, including ResGroups, User BeanCounters, NSProxy
cgroups, and others. These all need the basic abstraction of being
able to group together multiple processes in an aggregate, in order to
track/limit the resources permitted to those processes, or control
other behaviour of the processes, and all implement this grouping in
different ways.
This patchset provides a framework for tracking and grouping processes
into arbitrary "cgroups" and assigning arbitrary state to those
groupings, in order to control the behaviour of the cgroup as an
aggregate.
The intention is that the various resource management and
virtualization/cgroup efforts can also become task cgroup
clients, with the result that:
- the userspace APIs are (somewhat) normalised
- it's easier to test e.g. the ResGroups CPU controller in
conjunction with the BeanCounters memory controller, or use either of
them as the resource-control portion of a virtual server system.
- the additional kernel footprint of any of the competing resource
management systems is substantially reduced, since it doesn't need
to provide process grouping/containment, hence improving their
chances of getting into the kernel
This patch:
Add the main task cgroups framework - the cgroup filesystem, and the
basic structures for tracking membership and associating subsystem state
objects to tasks.
Signed-off-by: Paul Menage <menage@google.com>
Cc: Serge E. Hallyn <serue@us.ibm.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: Paul Jackson <pj@sgi.com>
Cc: Kirill Korotaev <dev@openvz.org>
Cc: Herbert Poetzl <herbert@13thfloor.at>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Cc: Cedric Le Goater <clg@fr.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-19 14:39:30 +08:00
|
|
|
#ifdef CONFIG_CGROUPS
|
2007-10-19 14:39:36 +08:00
|
|
|
/* Control Group info protected by css_set_lock */
|
|
|
|
|
struct css_set *cgroups;
|
|
|
|
|
/* cg_list protected by css_set_lock and tsk->alloc_lock */
|
|
|
|
|
struct list_head cg_list;
|
Task Control Groups: basic task cgroup framework
Generic Process Control Groups
--------------------------
There have recently been various proposals floating around for
resource management/accounting and other task grouping subsystems in
the kernel, including ResGroups, User BeanCounters, NSProxy
cgroups, and others. These all need the basic abstraction of being
able to group together multiple processes in an aggregate, in order to
track/limit the resources permitted to those processes, or control
other behaviour of the processes, and all implement this grouping in
different ways.
This patchset provides a framework for tracking and grouping processes
into arbitrary "cgroups" and assigning arbitrary state to those
groupings, in order to control the behaviour of the cgroup as an
aggregate.
The intention is that the various resource management and
virtualization/cgroup efforts can also become task cgroup
clients, with the result that:
- the userspace APIs are (somewhat) normalised
- it's easier to test e.g. the ResGroups CPU controller in
conjunction with the BeanCounters memory controller, or use either of
them as the resource-control portion of a virtual server system.
- the additional kernel footprint of any of the competing resource
management systems is substantially reduced, since it doesn't need
to provide process grouping/containment, hence improving their
chances of getting into the kernel
This patch:
Add the main task cgroups framework - the cgroup filesystem, and the
basic structures for tracking membership and associating subsystem state
objects to tasks.
Signed-off-by: Paul Menage <menage@google.com>
Cc: Serge E. Hallyn <serue@us.ibm.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: Paul Jackson <pj@sgi.com>
Cc: Kirill Korotaev <dev@openvz.org>
Cc: Herbert Poetzl <herbert@13thfloor.at>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Cc: Cedric Le Goater <clg@fr.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-19 14:39:30 +08:00
|
|
|
#endif
|
2007-10-17 14:27:30 +08:00
|
|
|
#ifdef CONFIG_FUTEX
|
2006-03-27 17:16:22 +08:00
|
|
|
struct robust_list_head __user *robust_list;
|
2006-03-27 17:16:24 +08:00
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
struct compat_robust_list_head __user *compat_robust_list;
|
|
|
|
|
#endif
|
2006-06-27 17:54:58 +08:00
|
|
|
struct list_head pi_state_list;
|
|
|
|
|
struct futex_pi_state *pi_state_cache;
|
2008-05-15 19:09:15 +08:00
|
|
|
#endif
|
perf: Do the big rename: Performance Counters -> Performance Events
Bye-bye Performance Counters, welcome Performance Events!
In the past few months the perfcounters subsystem has grown out its
initial role of counting hardware events, and has become (and is
becoming) a much broader generic event enumeration, reporting, logging,
monitoring, analysis facility.
Naming its core object 'perf_counter' and naming the subsystem
'perfcounters' has become more and more of a misnomer. With pending
code like hw-breakpoints support the 'counter' name is less and
less appropriate.
All in one, we've decided to rename the subsystem to 'performance
events' and to propagate this rename through all fields, variables
and API names. (in an ABI compatible fashion)
The word 'event' is also a bit shorter than 'counter' - which makes
it slightly more convenient to write/handle as well.
Thanks goes to Stephane Eranian who first observed this misnomer and
suggested a rename.
User-space tooling and ABI compatibility is not affected - this patch
should be function-invariant. (Also, defconfigs were not touched to
keep the size down.)
This patch has been generated via the following script:
FILES=$(find * -type f | grep -vE 'oprofile|[^K]config')
sed -i \
-e 's/PERF_EVENT_/PERF_RECORD_/g' \
-e 's/PERF_COUNTER/PERF_EVENT/g' \
-e 's/perf_counter/perf_event/g' \
-e 's/nb_counters/nb_events/g' \
-e 's/swcounter/swevent/g' \
-e 's/tpcounter_event/tp_event/g' \
$FILES
for N in $(find . -name perf_counter.[ch]); do
M=$(echo $N | sed 's/perf_counter/perf_event/g')
mv $N $M
done
FILES=$(find . -name perf_event.*)
sed -i \
-e 's/COUNTER_MASK/REG_MASK/g' \
-e 's/COUNTER/EVENT/g' \
-e 's/\<event\>/event_id/g' \
-e 's/counter/event/g' \
-e 's/Counter/Event/g' \
$FILES
... to keep it as correct as possible. This script can also be
used by anyone who has pending perfcounters patches - it converts
a Linux kernel tree over to the new naming. We tried to time this
change to the point in time where the amount of pending patches
is the smallest: the end of the merge window.
Namespace clashes were fixed up in a preparatory patch - and some
stylistic fallout will be fixed up in a subsequent patch.
( NOTE: 'counters' are still the proper terminology when we deal
with hardware registers - and these sed scripts are a bit
over-eager in renaming them. I've undone some of that, but
in case there's something left where 'counter' would be
better than 'event' we can undo that on an individual basis
instead of touching an otherwise nicely automated patch. )
Suggested-by: Stephane Eranian <eranian@google.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Paul Mackerras <paulus@samba.org>
Reviewed-by: Arjan van de Ven <arjan@linux.intel.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Howells <dhowells@redhat.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: <linux-arch@vger.kernel.org>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-21 18:02:48 +08:00
|
|
|
#ifdef CONFIG_PERF_EVENTS
|
|
|
|
|
struct perf_event_context *perf_event_ctxp;
|
|
|
|
|
struct mutex perf_event_mutex;
|
|
|
|
|
struct list_head perf_event_list;
|
perf_counter: Dynamically allocate tasks' perf_counter_context struct
This replaces the struct perf_counter_context in the task_struct with
a pointer to a dynamically allocated perf_counter_context struct. The
main reason for doing is this is to allow us to transfer a
perf_counter_context from one task to another when we do lazy PMU
switching in a later patch.
This has a few side-benefits: the task_struct becomes a little smaller,
we save some memory because only tasks that have perf_counters attached
get a perf_counter_context allocated for them, and we can remove the
inclusion of <linux/perf_counter.h> in sched.h, meaning that we don't
end up recompiling nearly everything whenever perf_counter.h changes.
The perf_counter_context structures are reference-counted and freed
when the last reference is dropped. A context can have references
from its task and the counters on its task. Counters can outlive the
task so it is possible that a context will be freed well after its
task has exited.
Contexts are allocated on fork if the parent had a context, or
otherwise the first time that a per-task counter is created on a task.
In the latter case, we set the context pointer in the task struct
locklessly using an atomic compare-and-exchange operation in case we
raced with some other task in creating a context for the subject task.
This also removes the task pointer from the perf_counter struct. The
task pointer was not used anywhere and would make it harder to move a
context from one task to another. Anything that needed to know which
task a counter was attached to was already using counter->ctx->task.
The __perf_counter_init_context function moves up in perf_counter.c
so that it can be called from find_get_context, and now initializes
the refcount, but is otherwise unchanged.
We were potentially calling list_del_counter twice: once from
__perf_counter_exit_task when the task exits and once from
__perf_counter_remove_from_context when the counter's fd gets closed.
This adds a check in list_del_counter so it doesn't do anything if
the counter has already been removed from the lists.
Since perf_counter_task_sched_in doesn't do anything if the task doesn't
have a context, and leaves cpuctx->task_ctx = NULL, this adds code to
__perf_install_in_context to set cpuctx->task_ctx if necessary, i.e. in
the case where the current task adds the first counter to itself and
thus creates a context for itself.
This also adds similar code to __perf_counter_enable to handle a
similar situation which can arise when the counters have been disabled
using prctl; that also leaves cpuctx->task_ctx = NULL.
[ Impact: refactor counter context management to prepare for new feature ]
Signed-off-by: Paul Mackerras <paulus@samba.org>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Corey Ashford <cjashfor@linux.vnet.ibm.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
LKML-Reference: <18966.10075.781053.231153@cargo.ozlabs.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-05-22 12:17:31 +08:00
|
|
|
#endif
|
2008-05-15 19:09:15 +08:00
|
|
|
#ifdef CONFIG_NUMA
|
cpuset,mm: update tasks' mems_allowed in time
Fix allocating page cache/slab object on the unallowed node when memory
spread is set by updating tasks' mems_allowed after its cpuset's mems is
changed.
In order to update tasks' mems_allowed in time, we must modify the code of
memory policy. Because the memory policy is applied in the process's
context originally. After applying this patch, one task directly
manipulates anothers mems_allowed, and we use alloc_lock in the
task_struct to protect mems_allowed and memory policy of the task.
But in the fast path, we didn't use lock to protect them, because adding a
lock may lead to performance regression. But if we don't add a lock,the
task might see no nodes when changing cpuset's mems_allowed to some
non-overlapping set. In order to avoid it, we set all new allowed nodes,
then clear newly disallowed ones.
[lee.schermerhorn@hp.com:
The rework of mpol_new() to extract the adjusting of the node mask to
apply cpuset and mpol flags "context" breaks set_mempolicy() and mbind()
with MPOL_PREFERRED and a NULL nodemask--i.e., explicit local
allocation. Fix this by adding the check for MPOL_PREFERRED and empty
node mask to mpol_new_mpolicy().
Remove the now unneeded 'nodes = NULL' from mpol_new().
Note that mpol_new_mempolicy() is always called with a non-NULL
'nodes' parameter now that it has been removed from mpol_new().
Therefore, we don't need to test nodes for NULL before testing it for
'empty'. However, just to be extra paranoid, add a VM_BUG_ON() to
verify this assumption.]
[lee.schermerhorn@hp.com:
I don't think the function name 'mpol_new_mempolicy' is descriptive
enough to differentiate it from mpol_new().
This function applies cpuset set context, usually constraining nodes
to those allowed by the cpuset. However, when the 'RELATIVE_NODES flag
is set, it also translates the nodes. So I settled on
'mpol_set_nodemask()', because the comment block for mpol_new() mentions
that we need to call this function to "set nodes".
Some additional minor line length, whitespace and typo cleanup.]
Signed-off-by: Miao Xie <miaox@cn.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Paul Menage <menage@google.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-06-17 06:31:49 +08:00
|
|
|
struct mempolicy *mempolicy; /* Protected by alloc_lock */
|
2008-05-15 19:09:15 +08:00
|
|
|
short il_next;
|
2007-10-17 14:27:30 +08:00
|
|
|
#endif
|
2005-06-27 16:55:12 +08:00
|
|
|
atomic_t fs_excl; /* holding fs exclusive resources */
|
2006-01-08 17:01:37 +08:00
|
|
|
struct rcu_head rcu;
|
2006-04-11 19:52:07 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* cache last used pipe for splice
|
|
|
|
|
*/
|
|
|
|
|
struct pipe_inode_info *splice_pipe;
|
2006-07-14 15:24:36 +08:00
|
|
|
#ifdef CONFIG_TASK_DELAY_ACCT
|
|
|
|
|
struct task_delay_info *delays;
|
2006-12-08 18:39:47 +08:00
|
|
|
#endif
|
|
|
|
|
#ifdef CONFIG_FAULT_INJECTION
|
|
|
|
|
int make_it_fail;
|
2006-07-14 15:24:36 +08:00
|
|
|
#endif
|
2007-10-17 14:25:50 +08:00
|
|
|
struct prop_local_single dirties;
|
2008-01-26 04:08:34 +08:00
|
|
|
#ifdef CONFIG_LATENCYTOP
|
|
|
|
|
int latency_record_count;
|
|
|
|
|
struct latency_record latency_record[LT_SAVECOUNT];
|
|
|
|
|
#endif
|
2008-09-02 06:52:40 +08:00
|
|
|
/*
|
|
|
|
|
* time slack values; these are used to round up poll() and
|
|
|
|
|
* select() etc timeout values. These are in nanoseconds.
|
|
|
|
|
*/
|
|
|
|
|
unsigned long timer_slack_ns;
|
|
|
|
|
unsigned long default_timer_slack_ns;
|
2008-11-06 16:37:40 +08:00
|
|
|
|
|
|
|
|
struct list_head *scm_work_list;
|
2008-11-26 04:07:04 +08:00
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
tree-wide: Assorted spelling fixes
In particular, several occurances of funny versions of 'success',
'unknown', 'therefore', 'acknowledge', 'argument', 'achieve', 'address',
'beginning', 'desirable', 'separate' and 'necessary' are fixed.
Signed-off-by: Daniel Mack <daniel@caiaq.de>
Cc: Joe Perches <joe@perches.com>
Cc: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
2010-02-03 08:01:28 +08:00
|
|
|
/* Index of current stored address in ret_stack */
|
2008-11-23 13:22:56 +08:00
|
|
|
int curr_ret_stack;
|
|
|
|
|
/* Stack of return addresses for return function tracing */
|
|
|
|
|
struct ftrace_ret_stack *ret_stack;
|
2009-03-24 13:10:15 +08:00
|
|
|
/* time stamp for last schedule */
|
|
|
|
|
unsigned long long ftrace_timestamp;
|
2008-11-23 13:22:56 +08:00
|
|
|
/*
|
|
|
|
|
* Number of functions that haven't been traced
|
|
|
|
|
* because of depth overrun.
|
|
|
|
|
*/
|
|
|
|
|
atomic_t trace_overrun;
|
2008-12-06 10:43:41 +08:00
|
|
|
/* Pause for the tracing */
|
|
|
|
|
atomic_t tracing_graph_pause;
|
2008-11-23 13:22:56 +08:00
|
|
|
#endif
|
2008-12-04 04:36:57 +08:00
|
|
|
#ifdef CONFIG_TRACING
|
|
|
|
|
/* state flags for use by tracers */
|
|
|
|
|
unsigned long trace;
|
tracing: add same level recursion detection
The tracing infrastructure allows for recursion. That is, an interrupt
may interrupt the act of tracing an event, and that interrupt may very well
perform its own trace. This is a recursive trace, and is fine to do.
The problem arises when there is a bug, and the utility doing the trace
calls something that recurses back into the tracer. This recursion is not
caused by an external event like an interrupt, but by code that is not
expected to recurse. The result could be a lockup.
This patch adds a bitmask to the task structure that keeps track
of the trace recursion. To find the interrupt depth, the following
algorithm is used:
level = hardirq_count() + softirq_count() + in_nmi;
Here, level will be the depth of interrutps and softirqs, and even handles
the nmi. Then the corresponding bit is set in the recursion bitmask.
If the bit was already set, we know we had a recursion at the same level
and we warn about it and fail the writing to the buffer.
After the data has been committed to the buffer, we clear the bit.
No atomics are needed. The only races are with interrupts and they reset
the bitmask before returning anywy.
[ Impact: detect same irq level trace recursion ]
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2009-04-17 09:41:52 +08:00
|
|
|
/* bitmask of trace recursion */
|
|
|
|
|
unsigned long trace_recursion;
|
|
|
|
|
#endif /* CONFIG_TRACING */
|
2009-09-23 07:45:40 +08:00
|
|
|
unsigned long stack_start;
|
2009-12-16 08:47:03 +08:00
|
|
|
#ifdef CONFIG_CGROUP_MEM_RES_CTLR /* memcg uses this to do batch job */
|
|
|
|
|
struct memcg_batch_info {
|
|
|
|
|
int do_batch; /* incremented when batch uncharge started */
|
|
|
|
|
struct mem_cgroup *memcg; /* target memcg of uncharge */
|
|
|
|
|
unsigned long bytes; /* uncharged usage */
|
|
|
|
|
unsigned long memsw_bytes; /* uncharged mem+swap usage */
|
|
|
|
|
} memcg_batch;
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
};
|
|
|
|
|
|
2009-03-13 04:35:43 +08:00
|
|
|
/* Future-safe accessor for struct task_struct's cpus_allowed. */
|
2009-12-18 01:43:29 +08:00
|
|
|
#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
|
2009-03-13 04:35:43 +08:00
|
|
|
|
2007-07-10 00:51:59 +08:00
|
|
|
/*
|
|
|
|
|
* Priority of a process goes from 0..MAX_PRIO-1, valid RT
|
|
|
|
|
* priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH
|
|
|
|
|
* tasks are in the range MAX_RT_PRIO..MAX_PRIO-1. Priority
|
|
|
|
|
* values are inverted: lower p->prio value means higher priority.
|
|
|
|
|
*
|
|
|
|
|
* The MAX_USER_RT_PRIO value allows the actual maximum
|
|
|
|
|
* RT priority to be separate from the value exported to
|
|
|
|
|
* user-space. This allows kernel threads to set their
|
|
|
|
|
* priority to a value higher than any user task. Note:
|
|
|
|
|
* MAX_RT_PRIO must not be smaller than MAX_USER_RT_PRIO.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define MAX_USER_RT_PRIO 100
|
|
|
|
|
#define MAX_RT_PRIO MAX_USER_RT_PRIO
|
|
|
|
|
|
|
|
|
|
#define MAX_PRIO (MAX_RT_PRIO + 40)
|
|
|
|
|
#define DEFAULT_PRIO (MAX_RT_PRIO + 20)
|
|
|
|
|
|
|
|
|
|
static inline int rt_prio(int prio)
|
|
|
|
|
{
|
|
|
|
|
if (unlikely(prio < MAX_RT_PRIO))
|
|
|
|
|
return 1;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline int rt_task(struct task_struct *p)
|
2007-07-10 00:51:59 +08:00
|
|
|
{
|
|
|
|
|
return rt_prio(p->prio);
|
|
|
|
|
}
|
|
|
|
|
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline struct pid *task_pid(struct task_struct *task)
|
2006-10-02 17:17:09 +08:00
|
|
|
{
|
|
|
|
|
return task->pids[PIDTYPE_PID].pid;
|
|
|
|
|
}
|
|
|
|
|
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline struct pid *task_tgid(struct task_struct *task)
|
2006-10-02 17:17:09 +08:00
|
|
|
{
|
|
|
|
|
return task->group_leader->pids[PIDTYPE_PID].pid;
|
|
|
|
|
}
|
|
|
|
|
|
2009-04-03 07:58:35 +08:00
|
|
|
/*
|
|
|
|
|
* Without tasklist or rcu lock it is not safe to dereference
|
|
|
|
|
* the result of task_pgrp/task_session even if task == current,
|
|
|
|
|
* we can race with another thread doing sys_setsid/sys_setpgid.
|
|
|
|
|
*/
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline struct pid *task_pgrp(struct task_struct *task)
|
2006-10-02 17:17:09 +08:00
|
|
|
{
|
|
|
|
|
return task->group_leader->pids[PIDTYPE_PGID].pid;
|
|
|
|
|
}
|
|
|
|
|
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline struct pid *task_session(struct task_struct *task)
|
2006-10-02 17:17:09 +08:00
|
|
|
{
|
|
|
|
|
return task->group_leader->pids[PIDTYPE_SID].pid;
|
|
|
|
|
}
|
|
|
|
|
|
2007-10-19 14:40:06 +08:00
|
|
|
struct pid_namespace;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* the helpers to get the task's different pids as they are seen
|
|
|
|
|
* from various namespaces
|
|
|
|
|
*
|
|
|
|
|
* task_xid_nr() : global id, i.e. the id seen from the init namespace;
|
2008-02-08 20:19:15 +08:00
|
|
|
* task_xid_vnr() : virtual id, i.e. the id seen from the pid namespace of
|
|
|
|
|
* current.
|
2007-10-19 14:40:06 +08:00
|
|
|
* task_xid_nr_ns() : id seen from the ns specified;
|
|
|
|
|
*
|
|
|
|
|
* set_task_vxid() : assigns a virtual id to a task;
|
|
|
|
|
*
|
|
|
|
|
* see also pid_nr() etc in include/linux/pid.h
|
|
|
|
|
*/
|
pids: refactor vnr/nr_ns helpers to make them safe
Inho, the safety rules for vnr/nr_ns helpers are horrible and buggy.
task_pid_nr_ns(task) needs rcu/tasklist depending on task == current.
As for "special" pids, vnr/nr_ns helpers always need rcu. However, if
task != current, they are unsafe even under rcu lock, we can't trust
task->group_leader without the special checks.
And almost every helper has a callsite which needs a fix.
Also, it is a bit annoying that the implementations of, say,
task_pgrp_vnr() and task_pgrp_nr_ns() are not "symmetrical".
This patch introduces the new helper, __task_pid_nr_ns(), which is always
safe to use, and turns all other helpers into the trivial wrappers.
After this I'll send another patch which converts task_tgid_xxx() as well,
they're are a bit special.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Louis Rilling <Louis.Rilling@kerlabs.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-03 07:58:38 +08:00
|
|
|
pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type,
|
|
|
|
|
struct pid_namespace *ns);
|
2007-10-19 14:40:06 +08:00
|
|
|
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline pid_t task_pid_nr(struct task_struct *tsk)
|
2007-10-19 14:40:06 +08:00
|
|
|
{
|
|
|
|
|
return tsk->pid;
|
|
|
|
|
}
|
|
|
|
|
|
pids: refactor vnr/nr_ns helpers to make them safe
Inho, the safety rules for vnr/nr_ns helpers are horrible and buggy.
task_pid_nr_ns(task) needs rcu/tasklist depending on task == current.
As for "special" pids, vnr/nr_ns helpers always need rcu. However, if
task != current, they are unsafe even under rcu lock, we can't trust
task->group_leader without the special checks.
And almost every helper has a callsite which needs a fix.
Also, it is a bit annoying that the implementations of, say,
task_pgrp_vnr() and task_pgrp_nr_ns() are not "symmetrical".
This patch introduces the new helper, __task_pid_nr_ns(), which is always
safe to use, and turns all other helpers into the trivial wrappers.
After this I'll send another patch which converts task_tgid_xxx() as well,
they're are a bit special.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Louis Rilling <Louis.Rilling@kerlabs.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-03 07:58:38 +08:00
|
|
|
static inline pid_t task_pid_nr_ns(struct task_struct *tsk,
|
|
|
|
|
struct pid_namespace *ns)
|
|
|
|
|
{
|
|
|
|
|
return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
|
|
|
|
|
}
|
2007-10-19 14:40:06 +08:00
|
|
|
|
|
|
|
|
static inline pid_t task_pid_vnr(struct task_struct *tsk)
|
|
|
|
|
{
|
pids: refactor vnr/nr_ns helpers to make them safe
Inho, the safety rules for vnr/nr_ns helpers are horrible and buggy.
task_pid_nr_ns(task) needs rcu/tasklist depending on task == current.
As for "special" pids, vnr/nr_ns helpers always need rcu. However, if
task != current, they are unsafe even under rcu lock, we can't trust
task->group_leader without the special checks.
And almost every helper has a callsite which needs a fix.
Also, it is a bit annoying that the implementations of, say,
task_pgrp_vnr() and task_pgrp_nr_ns() are not "symmetrical".
This patch introduces the new helper, __task_pid_nr_ns(), which is always
safe to use, and turns all other helpers into the trivial wrappers.
After this I'll send another patch which converts task_tgid_xxx() as well,
they're are a bit special.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Louis Rilling <Louis.Rilling@kerlabs.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-03 07:58:38 +08:00
|
|
|
return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
|
2007-10-19 14:40:06 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline pid_t task_tgid_nr(struct task_struct *tsk)
|
2007-10-19 14:40:06 +08:00
|
|
|
{
|
|
|
|
|
return tsk->tgid;
|
|
|
|
|
}
|
|
|
|
|
|
2007-10-19 14:40:19 +08:00
|
|
|
pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns);
|
2007-10-19 14:40:06 +08:00
|
|
|
|
|
|
|
|
static inline pid_t task_tgid_vnr(struct task_struct *tsk)
|
|
|
|
|
{
|
|
|
|
|
return pid_vnr(task_tgid(tsk));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
pids: refactor vnr/nr_ns helpers to make them safe
Inho, the safety rules for vnr/nr_ns helpers are horrible and buggy.
task_pid_nr_ns(task) needs rcu/tasklist depending on task == current.
As for "special" pids, vnr/nr_ns helpers always need rcu. However, if
task != current, they are unsafe even under rcu lock, we can't trust
task->group_leader without the special checks.
And almost every helper has a callsite which needs a fix.
Also, it is a bit annoying that the implementations of, say,
task_pgrp_vnr() and task_pgrp_nr_ns() are not "symmetrical".
This patch introduces the new helper, __task_pid_nr_ns(), which is always
safe to use, and turns all other helpers into the trivial wrappers.
After this I'll send another patch which converts task_tgid_xxx() as well,
they're are a bit special.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Louis Rilling <Louis.Rilling@kerlabs.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-03 07:58:38 +08:00
|
|
|
static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk,
|
|
|
|
|
struct pid_namespace *ns)
|
2007-10-19 14:40:06 +08:00
|
|
|
{
|
pids: refactor vnr/nr_ns helpers to make them safe
Inho, the safety rules for vnr/nr_ns helpers are horrible and buggy.
task_pid_nr_ns(task) needs rcu/tasklist depending on task == current.
As for "special" pids, vnr/nr_ns helpers always need rcu. However, if
task != current, they are unsafe even under rcu lock, we can't trust
task->group_leader without the special checks.
And almost every helper has a callsite which needs a fix.
Also, it is a bit annoying that the implementations of, say,
task_pgrp_vnr() and task_pgrp_nr_ns() are not "symmetrical".
This patch introduces the new helper, __task_pid_nr_ns(), which is always
safe to use, and turns all other helpers into the trivial wrappers.
After this I'll send another patch which converts task_tgid_xxx() as well,
they're are a bit special.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Louis Rilling <Louis.Rilling@kerlabs.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-03 07:58:38 +08:00
|
|
|
return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
|
2007-10-19 14:40:06 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
|
|
|
|
|
{
|
pids: refactor vnr/nr_ns helpers to make them safe
Inho, the safety rules for vnr/nr_ns helpers are horrible and buggy.
task_pid_nr_ns(task) needs rcu/tasklist depending on task == current.
As for "special" pids, vnr/nr_ns helpers always need rcu. However, if
task != current, they are unsafe even under rcu lock, we can't trust
task->group_leader without the special checks.
And almost every helper has a callsite which needs a fix.
Also, it is a bit annoying that the implementations of, say,
task_pgrp_vnr() and task_pgrp_nr_ns() are not "symmetrical".
This patch introduces the new helper, __task_pid_nr_ns(), which is always
safe to use, and turns all other helpers into the trivial wrappers.
After this I'll send another patch which converts task_tgid_xxx() as well,
they're are a bit special.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Louis Rilling <Louis.Rilling@kerlabs.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-03 07:58:38 +08:00
|
|
|
return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
|
2007-10-19 14:40:06 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
pids: refactor vnr/nr_ns helpers to make them safe
Inho, the safety rules for vnr/nr_ns helpers are horrible and buggy.
task_pid_nr_ns(task) needs rcu/tasklist depending on task == current.
As for "special" pids, vnr/nr_ns helpers always need rcu. However, if
task != current, they are unsafe even under rcu lock, we can't trust
task->group_leader without the special checks.
And almost every helper has a callsite which needs a fix.
Also, it is a bit annoying that the implementations of, say,
task_pgrp_vnr() and task_pgrp_nr_ns() are not "symmetrical".
This patch introduces the new helper, __task_pid_nr_ns(), which is always
safe to use, and turns all other helpers into the trivial wrappers.
After this I'll send another patch which converts task_tgid_xxx() as well,
they're are a bit special.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Louis Rilling <Louis.Rilling@kerlabs.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-03 07:58:38 +08:00
|
|
|
static inline pid_t task_session_nr_ns(struct task_struct *tsk,
|
|
|
|
|
struct pid_namespace *ns)
|
2007-10-19 14:40:06 +08:00
|
|
|
{
|
pids: refactor vnr/nr_ns helpers to make them safe
Inho, the safety rules for vnr/nr_ns helpers are horrible and buggy.
task_pid_nr_ns(task) needs rcu/tasklist depending on task == current.
As for "special" pids, vnr/nr_ns helpers always need rcu. However, if
task != current, they are unsafe even under rcu lock, we can't trust
task->group_leader without the special checks.
And almost every helper has a callsite which needs a fix.
Also, it is a bit annoying that the implementations of, say,
task_pgrp_vnr() and task_pgrp_nr_ns() are not "symmetrical".
This patch introduces the new helper, __task_pid_nr_ns(), which is always
safe to use, and turns all other helpers into the trivial wrappers.
After this I'll send another patch which converts task_tgid_xxx() as well,
they're are a bit special.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Louis Rilling <Louis.Rilling@kerlabs.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-03 07:58:38 +08:00
|
|
|
return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
|
2007-10-19 14:40:06 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline pid_t task_session_vnr(struct task_struct *tsk)
|
|
|
|
|
{
|
pids: refactor vnr/nr_ns helpers to make them safe
Inho, the safety rules for vnr/nr_ns helpers are horrible and buggy.
task_pid_nr_ns(task) needs rcu/tasklist depending on task == current.
As for "special" pids, vnr/nr_ns helpers always need rcu. However, if
task != current, they are unsafe even under rcu lock, we can't trust
task->group_leader without the special checks.
And almost every helper has a callsite which needs a fix.
Also, it is a bit annoying that the implementations of, say,
task_pgrp_vnr() and task_pgrp_nr_ns() are not "symmetrical".
This patch introduces the new helper, __task_pid_nr_ns(), which is always
safe to use, and turns all other helpers into the trivial wrappers.
After this I'll send another patch which converts task_tgid_xxx() as well,
they're are a bit special.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Louis Rilling <Louis.Rilling@kerlabs.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Pavel Emelyanov <xemul@openvz.org>
Cc: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Cc: Roland McGrath <roland@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-04-03 07:58:38 +08:00
|
|
|
return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
|
2007-10-19 14:40:06 +08:00
|
|
|
}
|
|
|
|
|
|
2009-04-03 07:58:39 +08:00
|
|
|
/* obsolete, do not use */
|
|
|
|
|
static inline pid_t task_pgrp_nr(struct task_struct *tsk)
|
|
|
|
|
{
|
|
|
|
|
return task_pgrp_nr_ns(tsk, &init_pid_ns);
|
|
|
|
|
}
|
2007-10-19 14:40:06 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/**
|
|
|
|
|
* pid_alive - check that a task structure is not stale
|
|
|
|
|
* @p: Task structure to be checked.
|
|
|
|
|
*
|
|
|
|
|
* Test if a process is not yet dead (at most zombie state)
|
|
|
|
|
* If pid_alive fails, then pointers within the task structure
|
|
|
|
|
* can be stale and must not be dereferenced.
|
|
|
|
|
*/
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline int pid_alive(struct task_struct *p)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
[PATCH] pidhash: Refactor the pid hash table
Simplifies the code, reduces the need for 4 pid hash tables, and makes the
code more capable.
In the discussions I had with Oleg it was felt that to a large extent the
cleanup itself justified the work. With struct pid being dynamically
allocated meant we could create the hash table entry when the pid was
allocated and free the hash table entry when the pid was freed. Instead of
playing with the hash lists when ever a process would attach or detach to a
process.
For myself the fact that it gave what my previous task_ref patch gave for free
with simpler code was a big win. The problem is that if you hold a reference
to struct task_struct you lock in 10K of low memory. If you do that in a user
controllable way like /proc does, with an unprivileged but hostile user space
application with typical resource limits of 1000 fds and 100 processes I can
trigger the OOM killer by consuming all of low memory with task structs, on a
machine wight 1GB of low memory.
If I instead hold a reference to struct pid which holds a pointer to my
task_struct, I don't suffer from that problem because struct pid is 2 orders
of magnitude smaller. In fact struct pid is small enough that most other
kernel data structures dwarf it, so simply limiting the number of referring
data structures is enough to prevent exhaustion of low memory.
This splits the current struct pid into two structures, struct pid and struct
pid_link, and reduces our number of hash tables from PIDTYPE_MAX to just one.
struct pid_link is the per process linkage into the hash tables and lives in
struct task_struct. struct pid is given an indepedent lifetime, and holds
pointers to each of the pid types.
The independent life of struct pid simplifies attach_pid, and detach_pid,
because we are always manipulating the list of pids and not the hash table.
In addition in giving struct pid an indpendent life it makes the concept much
more powerful.
Kernel data structures can now embed a struct pid * instead of a pid_t and
not suffer from pid wrap around problems or from keeping unnecessarily
large amounts of memory allocated.
Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-03-31 18:31:42 +08:00
|
|
|
return p->pids[PIDTYPE_PID].pid != NULL;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2006-09-29 17:00:07 +08:00
|
|
|
/**
|
2007-10-19 14:39:52 +08:00
|
|
|
* is_global_init - check if a task structure is init
|
2006-10-06 15:44:01 +08:00
|
|
|
* @tsk: Task structure to be checked.
|
|
|
|
|
*
|
|
|
|
|
* Check if a task structure is the first user space task the kernel created.
|
2007-10-19 14:39:52 +08:00
|
|
|
*/
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline int is_global_init(struct task_struct *tsk)
|
2007-10-19 14:40:09 +08:00
|
|
|
{
|
|
|
|
|
return tsk->pid == 1;
|
|
|
|
|
}
|
2007-10-19 14:39:52 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* is_container_init:
|
|
|
|
|
* check whether in the task is init in its own pid namespace.
|
2006-09-29 17:00:07 +08:00
|
|
|
*/
|
2007-10-19 14:40:09 +08:00
|
|
|
extern int is_container_init(struct task_struct *tsk);
|
2006-09-29 17:00:07 +08:00
|
|
|
|
2006-10-02 17:19:00 +08:00
|
|
|
extern struct pid *cad_pid;
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
extern void free_task(struct task_struct *tsk);
|
|
|
|
|
#define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0)
|
2006-01-08 17:01:37 +08:00
|
|
|
|
2006-03-31 18:31:34 +08:00
|
|
|
extern void __put_task_struct(struct task_struct *t);
|
2006-01-08 17:01:37 +08:00
|
|
|
|
|
|
|
|
static inline void put_task_struct(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
if (atomic_dec_and_test(&t->usage))
|
2006-03-31 18:31:37 +08:00
|
|
|
__put_task_struct(t);
|
2006-01-08 17:01:37 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-11-26 13:48:30 +08:00
|
|
|
extern void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st);
|
sched, cputime: Introduce thread_group_times()
This is a real fix for problem of utime/stime values decreasing
described in the thread:
http://lkml.org/lkml/2009/11/3/522
Now cputime is accounted in the following way:
- {u,s}time in task_struct are increased every time when the thread
is interrupted by a tick (timer interrupt).
- When a thread exits, its {u,s}time are added to signal->{u,s}time,
after adjusted by task_times().
- When all threads in a thread_group exits, accumulated {u,s}time
(and also c{u,s}time) in signal struct are added to c{u,s}time
in signal struct of the group's parent.
So {u,s}time in task struct are "raw" tick count, while
{u,s}time and c{u,s}time in signal struct are "adjusted" values.
And accounted values are used by:
- task_times(), to get cputime of a thread:
This function returns adjusted values that originates from raw
{u,s}time and scaled by sum_exec_runtime that accounted by CFS.
- thread_group_cputime(), to get cputime of a thread group:
This function returns sum of all {u,s}time of living threads in
the group, plus {u,s}time in the signal struct that is sum of
adjusted cputimes of all exited threads belonged to the group.
The problem is the return value of thread_group_cputime(),
because it is mixed sum of "raw" value and "adjusted" value:
group's {u,s}time = foreach(thread){{u,s}time} + exited({u,s}time)
This misbehavior can break {u,s}time monotonicity.
Assume that if there is a thread that have raw values greater
than adjusted values (e.g. interrupted by 1000Hz ticks 50 times
but only runs 45ms) and if it exits, cputime will decrease (e.g.
-5ms).
To fix this, we could do:
group's {u,s}time = foreach(t){task_times(t)} + exited({u,s}time)
But task_times() contains hard divisions, so applying it for
every thread should be avoided.
This patch fixes the above problem in the following way:
- Modify thread's exit (= __exit_signal()) not to use task_times().
It means {u,s}time in signal struct accumulates raw values instead
of adjusted values. As the result it makes thread_group_cputime()
to return pure sum of "raw" values.
- Introduce a new function thread_group_times(*task, *utime, *stime)
that converts "raw" values of thread_group_cputime() to "adjusted"
values, in same calculation procedure as task_times().
- Modify group's exit (= wait_task_zombie()) to use this introduced
thread_group_times(). It make c{u,s}time in signal struct to
have adjusted values like before this patch.
- Replace some thread_group_cputime() by thread_group_times().
This replacements are only applied where conveys the "adjusted"
cputime to users, and where already uses task_times() near by it.
(i.e. sys_times(), getrusage(), and /proc/<PID>/stat.)
This patch have a positive side effect:
- Before this patch, if a group contains many short-life threads
(e.g. runs 0.9ms and not interrupted by ticks), the group's
cputime could be invisible since thread's cputime was accumulated
after adjusted: imagine adjustment function as adj(ticks, runtime),
{adj(0, 0.9) + adj(0, 0.9) + ....} = {0 + 0 + ....} = 0.
After this patch it will not happen because the adjustment is
applied after accumulated.
v2:
- remove if()s, put new variables into signal_struct.
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: Spencer Candland <spencer@bluehost.com>
Cc: Americo Wang <xiyou.wangcong@gmail.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: Stanislaw Gruszka <sgruszka@redhat.com>
LKML-Reference: <4B162517.8040909@jp.fujitsu.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-02 16:28:07 +08:00
|
|
|
extern void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st);
|
2008-09-06 00:12:23 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* Per process flags
|
|
|
|
|
*/
|
|
|
|
|
#define PF_ALIGNWARN 0x00000001 /* Print alignment warning msgs */
|
|
|
|
|
/* Not implemented yet, only for 486*/
|
|
|
|
|
#define PF_STARTING 0x00000002 /* being created */
|
|
|
|
|
#define PF_EXITING 0x00000004 /* getting shut down */
|
2007-06-09 04:47:00 +08:00
|
|
|
#define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */
|
2007-10-15 23:00:19 +08:00
|
|
|
#define PF_VCPU 0x00000010 /* I'm a virtual CPU */
|
2005-04-17 06:20:36 +08:00
|
|
|
#define PF_FORKNOEXEC 0x00000040 /* forked but didn't exec */
|
2009-09-16 17:50:14 +08:00
|
|
|
#define PF_MCE_PROCESS 0x00000080 /* process policy on mce errors */
|
2005-04-17 06:20:36 +08:00
|
|
|
#define PF_SUPERPRIV 0x00000100 /* used super-user privileges */
|
|
|
|
|
#define PF_DUMPCORE 0x00000200 /* dumped core */
|
|
|
|
|
#define PF_SIGNALED 0x00000400 /* killed by a signal */
|
|
|
|
|
#define PF_MEMALLOC 0x00000800 /* Allocating memory */
|
|
|
|
|
#define PF_FLUSHER 0x00001000 /* responsible for disk writeback */
|
|
|
|
|
#define PF_USED_MATH 0x00002000 /* if unset the fpu must be initialized before use */
|
2009-07-17 20:15:47 +08:00
|
|
|
#define PF_FREEZING 0x00004000 /* freeze in progress. do not account to load */
|
2005-04-17 06:20:36 +08:00
|
|
|
#define PF_NOFREEZE 0x00008000 /* this thread should not be frozen */
|
|
|
|
|
#define PF_FROZEN 0x00010000 /* frozen for system suspend */
|
|
|
|
|
#define PF_FSTRANS 0x00020000 /* inside a filesystem transaction */
|
|
|
|
|
#define PF_KSWAPD 0x00040000 /* I am kswapd */
|
2009-09-22 08:02:27 +08:00
|
|
|
#define PF_OOM_ORIGIN 0x00080000 /* Allocating much memory to others */
|
2005-04-17 06:20:36 +08:00
|
|
|
#define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */
|
2008-07-25 16:47:38 +08:00
|
|
|
#define PF_KTHREAD 0x00200000 /* I am a kernel thread */
|
2006-06-13 14:26:10 +08:00
|
|
|
#define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */
|
|
|
|
|
#define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */
|
|
|
|
|
#define PF_SPREAD_PAGE 0x01000000 /* Spread page cache over cpuset */
|
|
|
|
|
#define PF_SPREAD_SLAB 0x02000000 /* Spread some slab caches over cpuset */
|
2008-06-06 03:57:11 +08:00
|
|
|
#define PF_THREAD_BOUND 0x04000000 /* Thread bound to specific cpu */
|
2009-09-16 17:50:14 +08:00
|
|
|
#define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */
|
2006-03-24 19:16:08 +08:00
|
|
|
#define PF_MEMPOLICY 0x10000000 /* Non-default NUMA mempolicy */
|
2006-06-27 17:54:56 +08:00
|
|
|
#define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */
|
2007-05-24 04:57:25 +08:00
|
|
|
#define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezeable */
|
2008-06-12 04:04:29 +08:00
|
|
|
#define PF_FREEZER_NOSIG 0x80000000 /* Freezer won't send signals to it */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Only the _current_ task can read/write to tsk->flags, but other
|
|
|
|
|
* tasks can access tsk->flags in readonly mode for example
|
|
|
|
|
* with tsk_used_math (like during threaded core dumping).
|
|
|
|
|
* There is however an exception to this rule during ptrace
|
|
|
|
|
* or during fork: the ptracer task is allowed to write to the
|
|
|
|
|
* child->flags of its traced child (same goes for fork, the parent
|
|
|
|
|
* can write to the child->flags), because we're guaranteed the
|
|
|
|
|
* child is not running and in turn not changing child->flags
|
|
|
|
|
* at the same time the parent does it.
|
|
|
|
|
*/
|
|
|
|
|
#define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
|
|
|
|
|
#define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
|
|
|
|
|
#define clear_used_math() clear_stopped_child_used_math(current)
|
|
|
|
|
#define set_used_math() set_stopped_child_used_math(current)
|
|
|
|
|
#define conditional_stopped_child_used_math(condition, child) \
|
|
|
|
|
do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
|
|
|
|
|
#define conditional_used_math(condition) \
|
|
|
|
|
conditional_stopped_child_used_math(condition, current)
|
|
|
|
|
#define copy_to_stopped_child_used_math(child) \
|
|
|
|
|
do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
|
|
|
|
|
/* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
|
|
|
|
|
#define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
|
|
|
|
|
#define used_math() tsk_used_math(current)
|
|
|
|
|
|
rcu: Merge preemptable-RCU functionality into hierarchical RCU
Create a kernel/rcutree_plugin.h file that contains definitions
for preemptable RCU (or, under the #else branch of the #ifdef,
empty definitions for the classic non-preemptable semantics).
These definitions fit into plugins defined in kernel/rcutree.c
for this purpose.
This variant of preemptable RCU uses a new algorithm whose
read-side expense is roughly that of classic hierarchical RCU
under CONFIG_PREEMPT. This new algorithm's update-side expense
is similar to that of classic hierarchical RCU, and, in absence
of read-side preemption or blocking, is exactly that of classic
hierarchical RCU. Perhaps more important, this new algorithm
has a much simpler implementation, saving well over 1,000 lines
of code compared to mainline's implementation of preemptable
RCU, which will hopefully be retired in favor of this new
algorithm.
The simplifications are obtained by maintaining per-task
nesting state for running tasks, and using a simple
lock-protected algorithm to handle accounting when tasks block
within RCU read-side critical sections, making use of lessons
learned while creating numerous user-level RCU implementations
over the past 18 months.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: akpm@linux-foundation.org
Cc: mathieu.desnoyers@polymtl.ca
Cc: josht@linux.vnet.ibm.com
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
LKML-Reference: <12509746134003-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-08-23 04:56:52 +08:00
|
|
|
#ifdef CONFIG_TREE_PREEMPT_RCU
|
|
|
|
|
|
|
|
|
|
#define RCU_READ_UNLOCK_BLOCKED (1 << 0) /* blocked while in RCU read-side. */
|
|
|
|
|
#define RCU_READ_UNLOCK_NEED_QS (1 << 1) /* RCU core needs CPU response. */
|
|
|
|
|
|
|
|
|
|
static inline void rcu_copy_process(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
p->rcu_read_lock_nesting = 0;
|
|
|
|
|
p->rcu_read_unlock_special = 0;
|
2009-08-28 05:58:16 +08:00
|
|
|
p->rcu_blocked_node = NULL;
|
rcu: Merge preemptable-RCU functionality into hierarchical RCU
Create a kernel/rcutree_plugin.h file that contains definitions
for preemptable RCU (or, under the #else branch of the #ifdef,
empty definitions for the classic non-preemptable semantics).
These definitions fit into plugins defined in kernel/rcutree.c
for this purpose.
This variant of preemptable RCU uses a new algorithm whose
read-side expense is roughly that of classic hierarchical RCU
under CONFIG_PREEMPT. This new algorithm's update-side expense
is similar to that of classic hierarchical RCU, and, in absence
of read-side preemption or blocking, is exactly that of classic
hierarchical RCU. Perhaps more important, this new algorithm
has a much simpler implementation, saving well over 1,000 lines
of code compared to mainline's implementation of preemptable
RCU, which will hopefully be retired in favor of this new
algorithm.
The simplifications are obtained by maintaining per-task
nesting state for running tasks, and using a simple
lock-protected algorithm to handle accounting when tasks block
within RCU read-side critical sections, making use of lessons
learned while creating numerous user-level RCU implementations
over the past 18 months.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: akpm@linux-foundation.org
Cc: mathieu.desnoyers@polymtl.ca
Cc: josht@linux.vnet.ibm.com
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
LKML-Reference: <12509746134003-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-08-23 04:56:52 +08:00
|
|
|
INIT_LIST_HEAD(&p->rcu_node_entry);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
|
|
static inline void rcu_copy_process(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef CONFIG_SMP
|
2008-03-27 05:23:49 +08:00
|
|
|
extern int set_cpus_allowed_ptr(struct task_struct *p,
|
2008-11-25 00:05:14 +08:00
|
|
|
const struct cpumask *new_mask);
|
2005-04-17 06:20:36 +08:00
|
|
|
#else
|
2008-03-27 05:23:49 +08:00
|
|
|
static inline int set_cpus_allowed_ptr(struct task_struct *p,
|
2008-11-25 00:05:14 +08:00
|
|
|
const struct cpumask *new_mask)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2008-11-25 00:05:14 +08:00
|
|
|
if (!cpumask_test_cpu(0, new_mask))
|
2005-04-17 06:20:36 +08:00
|
|
|
return -EINVAL;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2009-09-24 23:34:38 +08:00
|
|
|
|
|
|
|
|
#ifndef CONFIG_CPUMASK_OFFSTACK
|
2008-03-27 05:23:49 +08:00
|
|
|
static inline int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask)
|
|
|
|
|
{
|
|
|
|
|
return set_cpus_allowed_ptr(p, &new_mask);
|
|
|
|
|
}
|
2009-09-24 23:34:38 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-02-27 03:20:29 +08:00
|
|
|
/*
|
|
|
|
|
* Architectures can set this to 1 if they have specified
|
|
|
|
|
* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK in their arch Kconfig,
|
|
|
|
|
* but then during bootup it turns out that sched_clock()
|
|
|
|
|
* is reliable after all:
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
|
|
|
|
|
extern int sched_clock_stable;
|
|
|
|
|
#endif
|
|
|
|
|
|
2009-12-10 09:07:03 +08:00
|
|
|
/* ftrace calls sched_clock() directly */
|
|
|
|
|
extern unsigned long long notrace sched_clock(void);
|
2007-07-20 03:28:35 +08:00
|
|
|
|
2008-08-11 14:59:03 +08:00
|
|
|
extern void sched_clock_init(void);
|
|
|
|
|
extern u64 sched_clock_cpu(int cpu);
|
2008-05-04 00:29:28 +08:00
|
|
|
|
2008-08-11 14:59:03 +08:00
|
|
|
#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
|
2008-05-04 00:29:28 +08:00
|
|
|
static inline void sched_clock_tick(void)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void sched_clock_idle_sleep_event(void)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
extern void sched_clock_tick(void);
|
|
|
|
|
extern void sched_clock_idle_sleep_event(void);
|
|
|
|
|
extern void sched_clock_idle_wakeup_event(u64 delta_ns);
|
|
|
|
|
#endif
|
|
|
|
|
|
2007-07-20 03:28:35 +08:00
|
|
|
/*
|
|
|
|
|
* For kernel-internal use: high-speed (but slightly incorrect) per-cpu
|
|
|
|
|
* clock constructed from sched_clock():
|
|
|
|
|
*/
|
|
|
|
|
extern unsigned long long cpu_clock(int cpu);
|
|
|
|
|
|
2006-07-03 15:25:41 +08:00
|
|
|
extern unsigned long long
|
2007-07-10 00:51:58 +08:00
|
|
|
task_sched_runtime(struct task_struct *task);
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
extern unsigned long long thread_group_sched_runtime(struct task_struct *task);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/* sched_exec is called by processes performing an exec */
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
extern void sched_exec(void);
|
|
|
|
|
#else
|
|
|
|
|
#define sched_exec() {}
|
|
|
|
|
#endif
|
|
|
|
|
|
2007-08-23 21:18:02 +08:00
|
|
|
extern void sched_clock_idle_sleep_event(void);
|
|
|
|
|
extern void sched_clock_idle_wakeup_event(u64 delta_ns);
|
2007-07-10 00:51:59 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
|
extern void idle_task_exit(void);
|
|
|
|
|
#else
|
|
|
|
|
static inline void idle_task_exit(void) {}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
extern void sched_idle_next(void);
|
2006-06-27 17:54:51 +08:00
|
|
|
|
2008-03-22 16:20:24 +08:00
|
|
|
#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
|
|
|
|
|
extern void wake_up_idle_cpu(int cpu);
|
|
|
|
|
#else
|
|
|
|
|
static inline void wake_up_idle_cpu(int cpu) { }
|
|
|
|
|
#endif
|
|
|
|
|
|
2007-08-26 00:41:53 +08:00
|
|
|
extern unsigned int sysctl_sched_latency;
|
2007-11-10 05:39:37 +08:00
|
|
|
extern unsigned int sysctl_sched_min_granularity;
|
2007-07-10 00:51:58 +08:00
|
|
|
extern unsigned int sysctl_sched_wakeup_granularity;
|
2008-12-30 02:09:17 +08:00
|
|
|
extern unsigned int sysctl_sched_shares_ratelimit;
|
|
|
|
|
extern unsigned int sysctl_sched_shares_thresh;
|
2007-07-10 00:51:58 +08:00
|
|
|
extern unsigned int sysctl_sched_child_runs_first;
|
2009-11-30 19:16:47 +08:00
|
|
|
|
|
|
|
|
enum sched_tunable_scaling {
|
|
|
|
|
SCHED_TUNABLESCALING_NONE,
|
|
|
|
|
SCHED_TUNABLESCALING_LOG,
|
|
|
|
|
SCHED_TUNABLESCALING_LINEAR,
|
|
|
|
|
SCHED_TUNABLESCALING_END,
|
|
|
|
|
};
|
|
|
|
|
extern enum sched_tunable_scaling sysctl_sched_tunable_scaling;
|
|
|
|
|
|
2009-09-09 21:41:37 +08:00
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
2007-10-15 23:00:18 +08:00
|
|
|
extern unsigned int sysctl_sched_migration_cost;
|
2007-11-10 05:39:39 +08:00
|
|
|
extern unsigned int sysctl_sched_nr_migrate;
|
2009-09-01 16:34:37 +08:00
|
|
|
extern unsigned int sysctl_sched_time_avg;
|
2009-04-16 14:45:34 +08:00
|
|
|
extern unsigned int sysctl_timer_migration;
|
2007-11-10 05:39:37 +08:00
|
|
|
|
2009-11-30 19:16:47 +08:00
|
|
|
int sched_proc_update_handler(struct ctl_table *table, int write,
|
2009-09-24 06:57:19 +08:00
|
|
|
void __user *buffer, size_t *length,
|
2007-11-10 05:39:37 +08:00
|
|
|
loff_t *ppos);
|
2007-10-15 23:00:02 +08:00
|
|
|
#endif
|
2009-04-16 14:46:41 +08:00
|
|
|
#ifdef CONFIG_SCHED_DEBUG
|
|
|
|
|
static inline unsigned int get_sysctl_timer_migration(void)
|
|
|
|
|
{
|
|
|
|
|
return sysctl_timer_migration;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
static inline unsigned int get_sysctl_timer_migration(void)
|
|
|
|
|
{
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2008-02-13 22:45:39 +08:00
|
|
|
extern unsigned int sysctl_sched_rt_period;
|
|
|
|
|
extern int sysctl_sched_rt_runtime;
|
2007-10-15 23:00:02 +08:00
|
|
|
|
2008-04-20 01:44:57 +08:00
|
|
|
int sched_rt_handler(struct ctl_table *table, int write,
|
2009-09-24 06:57:19 +08:00
|
|
|
void __user *buffer, size_t *lenp,
|
2008-04-20 01:44:57 +08:00
|
|
|
loff_t *ppos);
|
|
|
|
|
|
2007-10-15 23:00:02 +08:00
|
|
|
extern unsigned int sysctl_sched_compat_yield;
|
2007-07-10 00:51:58 +08:00
|
|
|
|
2006-06-27 17:54:51 +08:00
|
|
|
#ifdef CONFIG_RT_MUTEXES
|
2006-07-03 15:25:41 +08:00
|
|
|
extern int rt_mutex_getprio(struct task_struct *p);
|
|
|
|
|
extern void rt_mutex_setprio(struct task_struct *p, int prio);
|
|
|
|
|
extern void rt_mutex_adjust_pi(struct task_struct *p);
|
2006-06-27 17:54:51 +08:00
|
|
|
#else
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline int rt_mutex_getprio(struct task_struct *p)
|
2006-06-27 17:54:51 +08:00
|
|
|
{
|
|
|
|
|
return p->normal_prio;
|
|
|
|
|
}
|
2006-06-27 17:55:02 +08:00
|
|
|
# define rt_mutex_adjust_pi(p) do { } while (0)
|
2006-06-27 17:54:51 +08:00
|
|
|
#endif
|
|
|
|
|
|
2006-07-03 15:25:41 +08:00
|
|
|
extern void set_user_nice(struct task_struct *p, long nice);
|
|
|
|
|
extern int task_prio(const struct task_struct *p);
|
|
|
|
|
extern int task_nice(const struct task_struct *p);
|
|
|
|
|
extern int can_nice(const struct task_struct *p, const int nice);
|
|
|
|
|
extern int task_curr(const struct task_struct *p);
|
2005-04-17 06:20:36 +08:00
|
|
|
extern int idle_cpu(int cpu);
|
|
|
|
|
extern int sched_setscheduler(struct task_struct *, int, struct sched_param *);
|
2008-06-23 11:55:38 +08:00
|
|
|
extern int sched_setscheduler_nocheck(struct task_struct *, int,
|
|
|
|
|
struct sched_param *);
|
2006-07-03 15:25:41 +08:00
|
|
|
extern struct task_struct *idle_task(int cpu);
|
|
|
|
|
extern struct task_struct *curr_task(int cpu);
|
|
|
|
|
extern void set_curr_task(int cpu, struct task_struct *p);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
void yield(void);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The default (Linux) execution domain.
|
|
|
|
|
*/
|
|
|
|
|
extern struct exec_domain default_exec_domain;
|
|
|
|
|
|
|
|
|
|
union thread_union {
|
|
|
|
|
struct thread_info thread_info;
|
|
|
|
|
unsigned long stack[THREAD_SIZE/sizeof(long)];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
#ifndef __HAVE_ARCH_KSTACK_END
|
|
|
|
|
static inline int kstack_end(void *addr)
|
|
|
|
|
{
|
|
|
|
|
/* Reliable end of stack detection:
|
|
|
|
|
* Some APM bios versions misalign the stack
|
|
|
|
|
*/
|
|
|
|
|
return !(((unsigned long)addr+sizeof(void*)-1) & (THREAD_SIZE-sizeof(void*)));
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
extern union thread_union init_thread_union;
|
|
|
|
|
extern struct task_struct init_task;
|
|
|
|
|
|
|
|
|
|
extern struct mm_struct init_mm;
|
|
|
|
|
|
2007-10-19 14:40:06 +08:00
|
|
|
extern struct pid_namespace init_pid_ns;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* find a task by one of its numerical ids
|
|
|
|
|
*
|
|
|
|
|
* find_task_by_pid_ns():
|
|
|
|
|
* finds a task by its pid in the specified namespace
|
2007-10-19 14:40:16 +08:00
|
|
|
* find_task_by_vpid():
|
|
|
|
|
* finds a task by its virtual pid
|
2007-10-19 14:40:06 +08:00
|
|
|
*
|
2008-07-25 16:48:36 +08:00
|
|
|
* see also find_vpid() etc in include/linux/pid.h
|
2007-10-19 14:40:06 +08:00
|
|
|
*/
|
|
|
|
|
|
2007-10-19 14:40:16 +08:00
|
|
|
extern struct task_struct *find_task_by_vpid(pid_t nr);
|
|
|
|
|
extern struct task_struct *find_task_by_pid_ns(pid_t nr,
|
|
|
|
|
struct pid_namespace *ns);
|
2007-10-19 14:40:06 +08:00
|
|
|
|
2008-02-08 20:19:09 +08:00
|
|
|
extern void __set_special_pids(struct pid *pid);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/* per-UID process charging. */
|
2007-07-16 14:40:59 +08:00
|
|
|
extern struct user_struct * alloc_uid(struct user_namespace *, uid_t);
|
2005-04-17 06:20:36 +08:00
|
|
|
static inline struct user_struct *get_uid(struct user_struct *u)
|
|
|
|
|
{
|
|
|
|
|
atomic_inc(&u->__count);
|
|
|
|
|
return u;
|
|
|
|
|
}
|
|
|
|
|
extern void free_uid(struct user_struct *);
|
Fix user namespace exiting OOPs
It turned out, that the user namespace is released during the do_exit() in
exit_task_namespaces(), but the struct user_struct is released only during the
put_task_struct(), i.e. MUCH later.
On debug kernels with poisoned slabs this will cause the oops in
uid_hash_remove() because the head of the chain, which resides inside the
struct user_namespace, will be already freed and poisoned.
Since the uid hash itself is required only when someone can search it, i.e.
when the namespace is alive, we can safely unhash all the user_struct-s from
it during the namespace exiting. The subsequent free_uid() will complete the
user_struct destruction.
For example simple program
#include <sched.h>
char stack[2 * 1024 * 1024];
int f(void *foo)
{
return 0;
}
int main(void)
{
clone(f, stack + 1 * 1024 * 1024, 0x10000000, 0);
return 0;
}
run on kernel with CONFIG_USER_NS turned on will oops the
kernel immediately.
This was spotted during OpenVZ kernel testing.
Signed-off-by: Pavel Emelyanov <xemul@openvz.org>
Signed-off-by: Alexey Dobriyan <adobriyan@openvz.org>
Acked-by: "Serge E. Hallyn" <serue@us.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-09-19 13:46:45 +08:00
|
|
|
extern void release_uids(struct user_namespace *ns);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
#include <asm/current.h>
|
|
|
|
|
|
2006-09-29 17:00:32 +08:00
|
|
|
extern void do_timer(unsigned long ticks);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-02-14 07:03:15 +08:00
|
|
|
extern int wake_up_state(struct task_struct *tsk, unsigned int state);
|
|
|
|
|
extern int wake_up_process(struct task_struct *tsk);
|
|
|
|
|
extern void wake_up_new_task(struct task_struct *tsk,
|
|
|
|
|
unsigned long clone_flags);
|
2005-04-17 06:20:36 +08:00
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
extern void kick_process(struct task_struct *tsk);
|
|
|
|
|
#else
|
|
|
|
|
static inline void kick_process(struct task_struct *tsk) { }
|
|
|
|
|
#endif
|
2007-07-10 00:52:00 +08:00
|
|
|
extern void sched_fork(struct task_struct *p, int clone_flags);
|
|
|
|
|
extern void sched_dead(struct task_struct *p);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
extern void proc_caches_init(void);
|
|
|
|
|
extern void flush_signals(struct task_struct *);
|
2009-04-29 20:45:05 +08:00
|
|
|
extern void __flush_signals(struct task_struct *);
|
2007-05-09 17:34:37 +08:00
|
|
|
extern void ignore_signals(struct task_struct *);
|
2005-04-17 06:20:36 +08:00
|
|
|
extern void flush_signal_handlers(struct task_struct *, int force_default);
|
|
|
|
|
extern int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info);
|
|
|
|
|
|
|
|
|
|
static inline int dequeue_signal_lock(struct task_struct *tsk, sigset_t *mask, siginfo_t *info)
|
|
|
|
|
{
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
spin_lock_irqsave(&tsk->sighand->siglock, flags);
|
|
|
|
|
ret = dequeue_signal(tsk, mask, info);
|
|
|
|
|
spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
extern void block_all_signals(int (*notifier)(void *priv), void *priv,
|
|
|
|
|
sigset_t *mask);
|
|
|
|
|
extern void unblock_all_signals(void);
|
|
|
|
|
extern void release_task(struct task_struct * p);
|
|
|
|
|
extern int send_sig_info(int, struct siginfo *, struct task_struct *);
|
|
|
|
|
extern int force_sigsegv(int, struct task_struct *);
|
|
|
|
|
extern int force_sig_info(int, struct siginfo *, struct task_struct *);
|
2006-10-02 17:17:10 +08:00
|
|
|
extern int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp);
|
|
|
|
|
extern int kill_pid_info(int sig, struct siginfo *info, struct pid *pid);
|
2006-10-02 17:17:28 +08:00
|
|
|
extern int kill_pid_info_as_uid(int, struct siginfo *, struct pid *, uid_t, uid_t, u32);
|
2006-10-02 17:17:10 +08:00
|
|
|
extern int kill_pgrp(struct pid *pid, int sig, int priv);
|
|
|
|
|
extern int kill_pid(struct pid *pid, int sig, int priv);
|
2007-02-09 23:11:47 +08:00
|
|
|
extern int kill_proc_info(int, struct siginfo *, pid_t);
|
2008-07-26 10:45:54 +08:00
|
|
|
extern int do_notify_parent(struct task_struct *, int);
|
ptrace: __ptrace_detach: do __wake_up_parent() if we reap the tracee
The bug is old, it wasn't cause by recent changes.
Test case:
static void *tfunc(void *arg)
{
int pid = (long)arg;
assert(ptrace(PTRACE_ATTACH, pid, NULL, NULL) == 0);
kill(pid, SIGKILL);
sleep(1);
return NULL;
}
int main(void)
{
pthread_t th;
long pid = fork();
if (!pid)
pause();
signal(SIGCHLD, SIG_IGN);
assert(pthread_create(&th, NULL, tfunc, (void*)pid) == 0);
int r = waitpid(-1, NULL, __WNOTHREAD);
printf("waitpid: %d %m\n", r);
return 0;
}
Before the patch this program hangs, after this patch waitpid() correctly
fails with errno == -ECHILD.
The problem is, __ptrace_detach() reaps the EXIT_ZOMBIE tracee if its
->real_parent is our sub-thread and we ignore SIGCHLD. But in this case
we should wake up other threads which can sleep in do_wait().
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Vitaly Mayatskikh <vmayatsk@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-24 06:56:44 +08:00
|
|
|
extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent);
|
2005-04-17 06:20:36 +08:00
|
|
|
extern void force_sig(int, struct task_struct *);
|
|
|
|
|
extern int send_sig(int, struct task_struct *, int);
|
|
|
|
|
extern void zap_other_threads(struct task_struct *p);
|
|
|
|
|
extern struct sigqueue *sigqueue_alloc(void);
|
|
|
|
|
extern void sigqueue_free(struct sigqueue *);
|
2008-04-30 15:52:57 +08:00
|
|
|
extern int send_sigqueue(struct sigqueue *, struct task_struct *, int group);
|
2006-02-10 03:41:50 +08:00
|
|
|
extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *);
|
2005-04-17 06:20:36 +08:00
|
|
|
extern int do_sigaltstack(const stack_t __user *, stack_t __user *, unsigned long);
|
|
|
|
|
|
2006-10-02 17:19:00 +08:00
|
|
|
static inline int kill_cad_pid(int sig, int priv)
|
|
|
|
|
{
|
|
|
|
|
return kill_pid(cad_pid, sig, priv);
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* These can be the second arg to send_sig_info/send_group_sig_info. */
|
|
|
|
|
#define SEND_SIG_NOINFO ((struct siginfo *) 0)
|
|
|
|
|
#define SEND_SIG_PRIV ((struct siginfo *) 1)
|
|
|
|
|
#define SEND_SIG_FORCED ((struct siginfo *) 2)
|
|
|
|
|
|
2009-10-25 22:37:58 +08:00
|
|
|
/*
|
|
|
|
|
* True if we are on the alternate signal stack.
|
|
|
|
|
*/
|
2005-04-17 06:20:36 +08:00
|
|
|
static inline int on_sig_stack(unsigned long sp)
|
|
|
|
|
{
|
2009-10-25 22:37:58 +08:00
|
|
|
#ifdef CONFIG_STACK_GROWSUP
|
|
|
|
|
return sp >= current->sas_ss_sp &&
|
|
|
|
|
sp - current->sas_ss_sp < current->sas_ss_size;
|
|
|
|
|
#else
|
|
|
|
|
return sp > current->sas_ss_sp &&
|
|
|
|
|
sp - current->sas_ss_sp <= current->sas_ss_size;
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int sas_ss_flags(unsigned long sp)
|
|
|
|
|
{
|
|
|
|
|
return (current->sas_ss_size == 0 ? SS_DISABLE
|
|
|
|
|
: on_sig_stack(sp) ? SS_ONSTACK : 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Routines for handling mm_structs
|
|
|
|
|
*/
|
|
|
|
|
extern struct mm_struct * mm_alloc(void);
|
|
|
|
|
|
|
|
|
|
/* mmdrop drops the mm and the page tables */
|
2008-02-14 07:03:15 +08:00
|
|
|
extern void __mmdrop(struct mm_struct *);
|
2005-04-17 06:20:36 +08:00
|
|
|
static inline void mmdrop(struct mm_struct * mm)
|
|
|
|
|
{
|
2007-07-10 00:52:01 +08:00
|
|
|
if (unlikely(atomic_dec_and_test(&mm->mm_count)))
|
2005-04-17 06:20:36 +08:00
|
|
|
__mmdrop(mm);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* mmput gets rid of the mappings and all user-space */
|
|
|
|
|
extern void mmput(struct mm_struct *);
|
|
|
|
|
/* Grab a reference to a task's mm, if it is not already going away */
|
|
|
|
|
extern struct mm_struct *get_task_mm(struct task_struct *task);
|
|
|
|
|
/* Remove the current tasks stale references to the old mm_struct */
|
|
|
|
|
extern void mm_release(struct task_struct *, struct mm_struct *);
|
2008-03-26 01:47:10 +08:00
|
|
|
/* Allocate a new mm structure and copy contents from tsk->mm */
|
|
|
|
|
extern struct mm_struct *dup_mm(struct task_struct *tsk);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-04-03 07:56:59 +08:00
|
|
|
extern int copy_thread(unsigned long, unsigned long, unsigned long,
|
|
|
|
|
struct task_struct *, struct pt_regs *);
|
2005-04-17 06:20:36 +08:00
|
|
|
extern void flush_thread(void);
|
|
|
|
|
extern void exit_thread(void);
|
|
|
|
|
|
|
|
|
|
extern void exit_files(struct task_struct *);
|
2006-03-29 08:11:16 +08:00
|
|
|
extern void __cleanup_signal(struct signal_struct *);
|
2006-03-29 08:11:27 +08:00
|
|
|
extern void __cleanup_sighand(struct sighand_struct *);
|
2008-05-27 00:55:42 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
extern void exit_itimers(struct signal_struct *);
|
2008-05-27 00:55:42 +08:00
|
|
|
extern void flush_itimer_signals(void);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
extern NORET_TYPE void do_group_exit(int);
|
|
|
|
|
|
|
|
|
|
extern void daemonize(const char *, ...);
|
|
|
|
|
extern int allow_signal(int);
|
|
|
|
|
extern int disallow_signal(int);
|
|
|
|
|
|
|
|
|
|
extern int do_execve(char *, char __user * __user *, char __user * __user *, struct pt_regs *);
|
|
|
|
|
extern long do_fork(unsigned long, unsigned long, struct pt_regs *, unsigned long, int __user *, int __user *);
|
2006-07-03 15:25:41 +08:00
|
|
|
struct task_struct *fork_idle(int);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
extern void set_task_comm(struct task_struct *tsk, char *from);
|
2008-02-05 14:27:21 +08:00
|
|
|
extern char *get_task_comm(char *to, struct task_struct *tsk);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
2008-07-26 10:45:58 +08:00
|
|
|
extern unsigned long wait_task_inactive(struct task_struct *, long match_state);
|
2005-04-17 06:20:36 +08:00
|
|
|
#else
|
2008-07-26 10:45:58 +08:00
|
|
|
static inline unsigned long wait_task_inactive(struct task_struct *p,
|
|
|
|
|
long match_state)
|
|
|
|
|
{
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif
|
|
|
|
|
|
2009-04-15 02:17:16 +08:00
|
|
|
#define next_task(p) \
|
|
|
|
|
list_entry_rcu((p)->tasks.next, struct task_struct, tasks)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
#define for_each_process(p) \
|
|
|
|
|
for (p = &init_task ; (p = next_task(p)) != &init_task ; )
|
|
|
|
|
|
2009-07-10 09:48:23 +08:00
|
|
|
extern bool current_is_single_threaded(void);
|
CRED: Inaugurate COW credentials
Inaugurate copy-on-write credentials management. This uses RCU to manage the
credentials pointer in the task_struct with respect to accesses by other tasks.
A process may only modify its own credentials, and so does not need locking to
access or modify its own credentials.
A mutex (cred_replace_mutex) is added to the task_struct to control the effect
of PTRACE_ATTACHED on credential calculations, particularly with respect to
execve().
With this patch, the contents of an active credentials struct may not be
changed directly; rather a new set of credentials must be prepared, modified
and committed using something like the following sequence of events:
struct cred *new = prepare_creds();
int ret = blah(new);
if (ret < 0) {
abort_creds(new);
return ret;
}
return commit_creds(new);
There are some exceptions to this rule: the keyrings pointed to by the active
credentials may be instantiated - keyrings violate the COW rule as managing
COW keyrings is tricky, given that it is possible for a task to directly alter
the keys in a keyring in use by another task.
To help enforce this, various pointers to sets of credentials, such as those in
the task_struct, are declared const. The purpose of this is compile-time
discouragement of altering credentials through those pointers. Once a set of
credentials has been made public through one of these pointers, it may not be
modified, except under special circumstances:
(1) Its reference count may incremented and decremented.
(2) The keyrings to which it points may be modified, but not replaced.
The only safe way to modify anything else is to create a replacement and commit
using the functions described in Documentation/credentials.txt (which will be
added by a later patch).
This patch and the preceding patches have been tested with the LTP SELinux
testsuite.
This patch makes several logical sets of alteration:
(1) execve().
This now prepares and commits credentials in various places in the
security code rather than altering the current creds directly.
(2) Temporary credential overrides.
do_coredump() and sys_faccessat() now prepare their own credentials and
temporarily override the ones currently on the acting thread, whilst
preventing interference from other threads by holding cred_replace_mutex
on the thread being dumped.
This will be replaced in a future patch by something that hands down the
credentials directly to the functions being called, rather than altering
the task's objective credentials.
(3) LSM interface.
A number of functions have been changed, added or removed:
(*) security_capset_check(), ->capset_check()
(*) security_capset_set(), ->capset_set()
Removed in favour of security_capset().
(*) security_capset(), ->capset()
New. This is passed a pointer to the new creds, a pointer to the old
creds and the proposed capability sets. It should fill in the new
creds or return an error. All pointers, barring the pointer to the
new creds, are now const.
(*) security_bprm_apply_creds(), ->bprm_apply_creds()
Changed; now returns a value, which will cause the process to be
killed if it's an error.
(*) security_task_alloc(), ->task_alloc_security()
Removed in favour of security_prepare_creds().
(*) security_cred_free(), ->cred_free()
New. Free security data attached to cred->security.
(*) security_prepare_creds(), ->cred_prepare()
New. Duplicate any security data attached to cred->security.
(*) security_commit_creds(), ->cred_commit()
New. Apply any security effects for the upcoming installation of new
security by commit_creds().
(*) security_task_post_setuid(), ->task_post_setuid()
Removed in favour of security_task_fix_setuid().
(*) security_task_fix_setuid(), ->task_fix_setuid()
Fix up the proposed new credentials for setuid(). This is used by
cap_set_fix_setuid() to implicitly adjust capabilities in line with
setuid() changes. Changes are made to the new credentials, rather
than the task itself as in security_task_post_setuid().
(*) security_task_reparent_to_init(), ->task_reparent_to_init()
Removed. Instead the task being reparented to init is referred
directly to init's credentials.
NOTE! This results in the loss of some state: SELinux's osid no
longer records the sid of the thread that forked it.
(*) security_key_alloc(), ->key_alloc()
(*) security_key_permission(), ->key_permission()
Changed. These now take cred pointers rather than task pointers to
refer to the security context.
(4) sys_capset().
This has been simplified and uses less locking. The LSM functions it
calls have been merged.
(5) reparent_to_kthreadd().
This gives the current thread the same credentials as init by simply using
commit_thread() to point that way.
(6) __sigqueue_alloc() and switch_uid()
__sigqueue_alloc() can't stop the target task from changing its creds
beneath it, so this function gets a reference to the currently applicable
user_struct which it then passes into the sigqueue struct it returns if
successful.
switch_uid() is now called from commit_creds(), and possibly should be
folded into that. commit_creds() should take care of protecting
__sigqueue_alloc().
(7) [sg]et[ug]id() and co and [sg]et_current_groups.
The set functions now all use prepare_creds(), commit_creds() and
abort_creds() to build and check a new set of credentials before applying
it.
security_task_set[ug]id() is called inside the prepared section. This
guarantees that nothing else will affect the creds until we've finished.
The calling of set_dumpable() has been moved into commit_creds().
Much of the functionality of set_user() has been moved into
commit_creds().
The get functions all simply access the data directly.
(8) security_task_prctl() and cap_task_prctl().
security_task_prctl() has been modified to return -ENOSYS if it doesn't
want to handle a function, or otherwise return the return value directly
rather than through an argument.
Additionally, cap_task_prctl() now prepares a new set of credentials, even
if it doesn't end up using it.
(9) Keyrings.
A number of changes have been made to the keyrings code:
(a) switch_uid_keyring(), copy_keys(), exit_keys() and suid_keys() have
all been dropped and built in to the credentials functions directly.
They may want separating out again later.
(b) key_alloc() and search_process_keyrings() now take a cred pointer
rather than a task pointer to specify the security context.
(c) copy_creds() gives a new thread within the same thread group a new
thread keyring if its parent had one, otherwise it discards the thread
keyring.
(d) The authorisation key now points directly to the credentials to extend
the search into rather pointing to the task that carries them.
(e) Installing thread, process or session keyrings causes a new set of
credentials to be created, even though it's not strictly necessary for
process or session keyrings (they're shared).
(10) Usermode helper.
The usermode helper code now carries a cred struct pointer in its
subprocess_info struct instead of a new session keyring pointer. This set
of credentials is derived from init_cred and installed on the new process
after it has been cloned.
call_usermodehelper_setup() allocates the new credentials and
call_usermodehelper_freeinfo() discards them if they haven't been used. A
special cred function (prepare_usermodeinfo_creds()) is provided
specifically for call_usermodehelper_setup() to call.
call_usermodehelper_setkeys() adjusts the credentials to sport the
supplied keyring as the new session keyring.
(11) SELinux.
SELinux has a number of changes, in addition to those to support the LSM
interface changes mentioned above:
(a) selinux_setprocattr() no longer does its check for whether the
current ptracer can access processes with the new SID inside the lock
that covers getting the ptracer's SID. Whilst this lock ensures that
the check is done with the ptracer pinned, the result is only valid
until the lock is released, so there's no point doing it inside the
lock.
(12) is_single_threaded().
This function has been extracted from selinux_setprocattr() and put into
a file of its own in the lib/ directory as join_session_keyring() now
wants to use it too.
The code in SELinux just checked to see whether a task shared mm_structs
with other tasks (CLONE_VM), but that isn't good enough. We really want
to know if they're part of the same thread group (CLONE_THREAD).
(13) nfsd.
The NFS server daemon now has to use the COW credentials to set the
credentials it is going to use. It really needs to pass the credentials
down to the functions it calls, but it can't do that until other patches
in this series have been applied.
Signed-off-by: David Howells <dhowells@redhat.com>
Acked-by: James Morris <jmorris@namei.org>
Signed-off-by: James Morris <jmorris@namei.org>
2008-11-14 07:39:23 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* Careful: do_each_thread/while_each_thread is a double loop so
|
|
|
|
|
* 'break' will not work as expected - use goto instead.
|
|
|
|
|
*/
|
|
|
|
|
#define do_each_thread(g, t) \
|
|
|
|
|
for (g = t = &init_task ; (g = t = next_task(g)) != &init_task ; ) do
|
|
|
|
|
|
|
|
|
|
#define while_each_thread(g, t) \
|
|
|
|
|
while ((t = next_thread(t)) != g)
|
|
|
|
|
|
2006-04-11 07:16:49 +08:00
|
|
|
/* de_thread depends on thread_group_leader not being a pid based check */
|
|
|
|
|
#define thread_group_leader(p) (p == p->group_leader)
|
2005-04-17 06:20:36 +08:00
|
|
|
|
[PATCH] proc: readdir race fix (take 3)
The problem: An opendir, readdir, closedir sequence can fail to report
process ids that are continually in use throughout the sequence of system
calls. For this race to trigger the process that proc_pid_readdir stops at
must exit before readdir is called again.
This can cause ps to fail to report processes, and it is in violation of
posix guarantees and normal application expectations with respect to
readdir.
Currently there is no way to work around this problem in user space short
of providing a gargantuan buffer to user space so the directory read all
happens in on system call.
This patch implements the normal directory semantics for proc, that
guarantee that a directory entry that is neither created nor destroyed
while reading the directory entry will be returned. For directory that are
either created or destroyed during the readdir you may or may not see them.
Furthermore you may seek to a directory offset you have previously seen.
These are the guarantee that ext[23] provides and that posix requires, and
more importantly that user space expects. Plus it is a simple semantic to
implement reliable service. It is just a matter of calling readdir a
second time if you are wondering if something new has show up.
These better semantics are implemented by scanning through the pids in
numerical order and by making the file offset a pid plus a fixed offset.
The pid scan happens on the pid bitmap, which when you look at it is
remarkably efficient for a brute force algorithm. Given that a typical
cache line is 64 bytes and thus covers space for 64*8 == 200 pids. There
are only 40 cache lines for the entire 32K pid space. A typical system
will have 100 pids or more so this is actually fewer cache lines we have to
look at to scan a linked list, and the worst case of having to scan the
entire pid bitmap is pretty reasonable.
If we need something more efficient we can go to a more efficient data
structure for indexing the pids, but for now what we have should be
sufficient.
In addition this takes no additional locks and is actually less code than
what we are doing now.
Also another very subtle bug in this area has been fixed. It is possible
to catch a task in the middle of de_thread where a thread is assuming the
thread of it's thread group leader. This patch carefully handles that case
so if we hit it we don't fail to return the pid, that is undergoing the
de_thread dance.
Thanks to KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> for
providing the first fix, pointing this out and working on it.
[oleg@tv-sign.ru: fix it]
Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
Acked-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: Jean Delvare <jdelvare@suse.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-10-02 17:17:04 +08:00
|
|
|
/* Do to the insanities of de_thread it is possible for a process
|
|
|
|
|
* to have the pid of the thread group leader without actually being
|
|
|
|
|
* the thread group leader. For iteration through the pids in proc
|
|
|
|
|
* all we care about is that we have a task with the appropriate
|
|
|
|
|
* pid, we don't actually care if we have the right task.
|
|
|
|
|
*/
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline int has_group_leader_pid(struct task_struct *p)
|
[PATCH] proc: readdir race fix (take 3)
The problem: An opendir, readdir, closedir sequence can fail to report
process ids that are continually in use throughout the sequence of system
calls. For this race to trigger the process that proc_pid_readdir stops at
must exit before readdir is called again.
This can cause ps to fail to report processes, and it is in violation of
posix guarantees and normal application expectations with respect to
readdir.
Currently there is no way to work around this problem in user space short
of providing a gargantuan buffer to user space so the directory read all
happens in on system call.
This patch implements the normal directory semantics for proc, that
guarantee that a directory entry that is neither created nor destroyed
while reading the directory entry will be returned. For directory that are
either created or destroyed during the readdir you may or may not see them.
Furthermore you may seek to a directory offset you have previously seen.
These are the guarantee that ext[23] provides and that posix requires, and
more importantly that user space expects. Plus it is a simple semantic to
implement reliable service. It is just a matter of calling readdir a
second time if you are wondering if something new has show up.
These better semantics are implemented by scanning through the pids in
numerical order and by making the file offset a pid plus a fixed offset.
The pid scan happens on the pid bitmap, which when you look at it is
remarkably efficient for a brute force algorithm. Given that a typical
cache line is 64 bytes and thus covers space for 64*8 == 200 pids. There
are only 40 cache lines for the entire 32K pid space. A typical system
will have 100 pids or more so this is actually fewer cache lines we have to
look at to scan a linked list, and the worst case of having to scan the
entire pid bitmap is pretty reasonable.
If we need something more efficient we can go to a more efficient data
structure for indexing the pids, but for now what we have should be
sufficient.
In addition this takes no additional locks and is actually less code than
what we are doing now.
Also another very subtle bug in this area has been fixed. It is possible
to catch a task in the middle of de_thread where a thread is assuming the
thread of it's thread group leader. This patch carefully handles that case
so if we hit it we don't fail to return the pid, that is undergoing the
de_thread dance.
Thanks to KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> for
providing the first fix, pointing this out and working on it.
[oleg@tv-sign.ru: fix it]
Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
Acked-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: Jean Delvare <jdelvare@suse.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-10-02 17:17:04 +08:00
|
|
|
{
|
|
|
|
|
return p->pid == p->tgid;
|
|
|
|
|
}
|
|
|
|
|
|
2007-10-19 14:40:18 +08:00
|
|
|
static inline
|
|
|
|
|
int same_thread_group(struct task_struct *p1, struct task_struct *p2)
|
|
|
|
|
{
|
|
|
|
|
return p1->tgid == p2->tgid;
|
|
|
|
|
}
|
|
|
|
|
|
2006-07-03 15:25:41 +08:00
|
|
|
static inline struct task_struct *next_thread(const struct task_struct *p)
|
2006-03-29 08:11:25 +08:00
|
|
|
{
|
2009-04-15 02:17:16 +08:00
|
|
|
return list_entry_rcu(p->thread_group.next,
|
|
|
|
|
struct task_struct, thread_group);
|
2006-03-29 08:11:25 +08:00
|
|
|
}
|
|
|
|
|
|
2007-10-26 16:17:22 +08:00
|
|
|
static inline int thread_group_empty(struct task_struct *p)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2006-03-29 08:11:25 +08:00
|
|
|
return list_empty(&p->thread_group);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define delay_group_leader(p) \
|
|
|
|
|
(thread_group_leader(p) && !thread_group_empty(p))
|
|
|
|
|
|
2009-04-03 07:58:18 +08:00
|
|
|
static inline int task_detached(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
return p->exit_signal == -1;
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
2006-06-23 17:05:18 +08:00
|
|
|
* Protects ->fs, ->files, ->mm, ->group_info, ->comm, keyring
|
2005-06-27 16:55:12 +08:00
|
|
|
* subscriptions and synchronises with wait4(). Also used in procfs. Also
|
Task Control Groups: basic task cgroup framework
Generic Process Control Groups
--------------------------
There have recently been various proposals floating around for
resource management/accounting and other task grouping subsystems in
the kernel, including ResGroups, User BeanCounters, NSProxy
cgroups, and others. These all need the basic abstraction of being
able to group together multiple processes in an aggregate, in order to
track/limit the resources permitted to those processes, or control
other behaviour of the processes, and all implement this grouping in
different ways.
This patchset provides a framework for tracking and grouping processes
into arbitrary "cgroups" and assigning arbitrary state to those
groupings, in order to control the behaviour of the cgroup as an
aggregate.
The intention is that the various resource management and
virtualization/cgroup efforts can also become task cgroup
clients, with the result that:
- the userspace APIs are (somewhat) normalised
- it's easier to test e.g. the ResGroups CPU controller in
conjunction with the BeanCounters memory controller, or use either of
them as the resource-control portion of a virtual server system.
- the additional kernel footprint of any of the competing resource
management systems is substantially reduced, since it doesn't need
to provide process grouping/containment, hence improving their
chances of getting into the kernel
This patch:
Add the main task cgroups framework - the cgroup filesystem, and the
basic structures for tracking membership and associating subsystem state
objects to tasks.
Signed-off-by: Paul Menage <menage@google.com>
Cc: Serge E. Hallyn <serue@us.ibm.com>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Dave Hansen <haveblue@us.ibm.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Cc: Paul Jackson <pj@sgi.com>
Cc: Kirill Korotaev <dev@openvz.org>
Cc: Herbert Poetzl <herbert@13thfloor.at>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Cc: Cedric Le Goater <clg@fr.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-10-19 14:39:30 +08:00
|
|
|
* pins the final release of task.io_context. Also protects ->cpuset and
|
|
|
|
|
* ->cgroup.subsys[].
|
2005-04-17 06:20:36 +08:00
|
|
|
*
|
|
|
|
|
* Nests both inside and outside of read_lock(&tasklist_lock).
|
|
|
|
|
* It must not be nested with write_lock_irq(&tasklist_lock),
|
|
|
|
|
* neither inside nor outside.
|
|
|
|
|
*/
|
|
|
|
|
static inline void task_lock(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
spin_lock(&p->alloc_lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void task_unlock(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
spin_unlock(&p->alloc_lock);
|
|
|
|
|
}
|
|
|
|
|
|
2006-03-29 08:11:13 +08:00
|
|
|
extern struct sighand_struct *lock_task_sighand(struct task_struct *tsk,
|
|
|
|
|
unsigned long *flags);
|
|
|
|
|
|
|
|
|
|
static inline void unlock_task_sighand(struct task_struct *tsk,
|
|
|
|
|
unsigned long *flags)
|
|
|
|
|
{
|
|
|
|
|
spin_unlock_irqrestore(&tsk->sighand->siglock, *flags);
|
|
|
|
|
}
|
|
|
|
|
|
2005-11-14 08:06:57 +08:00
|
|
|
#ifndef __HAVE_THREAD_FUNCTIONS
|
|
|
|
|
|
2007-05-09 17:35:17 +08:00
|
|
|
#define task_thread_info(task) ((struct thread_info *)(task)->stack)
|
|
|
|
|
#define task_stack_page(task) ((task)->stack)
|
2005-11-14 08:06:55 +08:00
|
|
|
|
2005-11-14 08:06:56 +08:00
|
|
|
static inline void setup_thread_stack(struct task_struct *p, struct task_struct *org)
|
|
|
|
|
{
|
|
|
|
|
*task_thread_info(p) = *task_thread_info(org);
|
|
|
|
|
task_thread_info(p)->task = p;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long *end_of_stack(struct task_struct *p)
|
|
|
|
|
{
|
2007-05-09 17:35:17 +08:00
|
|
|
return (unsigned long *)(task_thread_info(p) + 1);
|
2005-11-14 08:06:56 +08:00
|
|
|
}
|
|
|
|
|
|
2005-11-14 08:06:57 +08:00
|
|
|
#endif
|
|
|
|
|
|
2008-07-24 12:26:53 +08:00
|
|
|
static inline int object_is_on_stack(void *obj)
|
|
|
|
|
{
|
|
|
|
|
void *stack = task_stack_page(current);
|
|
|
|
|
|
|
|
|
|
return (obj >= stack) && (obj < (stack + THREAD_SIZE));
|
|
|
|
|
}
|
|
|
|
|
|
2008-04-18 14:56:15 +08:00
|
|
|
extern void thread_info_cache_init(void);
|
|
|
|
|
|
2008-04-23 05:38:23 +08:00
|
|
|
#ifdef CONFIG_DEBUG_STACK_USAGE
|
|
|
|
|
static inline unsigned long stack_not_used(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
unsigned long *n = end_of_stack(p);
|
|
|
|
|
|
|
|
|
|
do { /* Skip over canary */
|
|
|
|
|
n++;
|
|
|
|
|
} while (!*n);
|
|
|
|
|
|
|
|
|
|
return (unsigned long)n - (unsigned long)end_of_stack(p);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* set thread flags in other task's structures
|
|
|
|
|
* - see asm/thread_info.h for TIF_xxxx flags available
|
|
|
|
|
*/
|
|
|
|
|
static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
|
|
|
|
|
{
|
2005-11-14 08:06:55 +08:00
|
|
|
set_ti_thread_flag(task_thread_info(tsk), flag);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
|
|
|
|
|
{
|
2005-11-14 08:06:55 +08:00
|
|
|
clear_ti_thread_flag(task_thread_info(tsk), flag);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
|
|
|
|
|
{
|
2005-11-14 08:06:55 +08:00
|
|
|
return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
|
|
|
|
|
{
|
2005-11-14 08:06:55 +08:00
|
|
|
return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
|
|
|
|
|
{
|
2005-11-14 08:06:55 +08:00
|
|
|
return test_ti_thread_flag(task_thread_info(tsk), flag);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void set_tsk_need_resched(struct task_struct *tsk)
|
|
|
|
|
{
|
|
|
|
|
set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void clear_tsk_need_resched(struct task_struct *tsk)
|
|
|
|
|
{
|
|
|
|
|
clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
|
|
|
|
|
}
|
|
|
|
|
|
2008-04-23 19:13:29 +08:00
|
|
|
static inline int test_tsk_need_resched(struct task_struct *tsk)
|
|
|
|
|
{
|
|
|
|
|
return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
|
|
|
|
|
}
|
|
|
|
|
|
2009-05-14 00:55:10 +08:00
|
|
|
static inline int restart_syscall(void)
|
|
|
|
|
{
|
|
|
|
|
set_tsk_thread_flag(current, TIF_SIGPENDING);
|
|
|
|
|
return -ERESTARTNOINTR;
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
static inline int signal_pending(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING));
|
|
|
|
|
}
|
2007-12-07 00:15:50 +08:00
|
|
|
|
2009-09-24 06:57:04 +08:00
|
|
|
static inline int __fatal_signal_pending(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
return unlikely(sigismember(&p->pending.signal, SIGKILL));
|
|
|
|
|
}
|
2007-12-07 00:15:50 +08:00
|
|
|
|
|
|
|
|
static inline int fatal_signal_pending(struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
return signal_pending(p) && __fatal_signal_pending(p);
|
|
|
|
|
}
|
|
|
|
|
|
sched: fix TASK_WAKEKILL vs SIGKILL race
schedule() has the special "TASK_INTERRUPTIBLE && signal_pending()" case,
this allows us to do
current->state = TASK_INTERRUPTIBLE;
schedule();
without fear to sleep with pending signal.
However, the code like
current->state = TASK_KILLABLE;
schedule();
is not right, schedule() doesn't take TASK_WAKEKILL into account. This means
that mutex_lock_killable(), wait_for_completion_killable(), down_killable(),
schedule_timeout_killable() can miss SIGKILL (and btw the second SIGKILL has
no effect).
Introduce the new helper, signal_pending_state(), and change schedule() to
use it. Hopefully it will have more users, that is why the task's state is
passed separately.
Note this "__TASK_STOPPED | __TASK_TRACED" check in signal_pending_state().
This is needed to preserve the current behaviour (ptrace_notify). I hope
this check will be removed soon, but this (afaics good) change needs the
separate discussion.
The fast path is "(state & (INTERRUPTIBLE | WAKEKILL)) + signal_pending(p)",
basically the same that schedule() does now. However, this patch of course
bloats schedule().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-06-09 01:20:41 +08:00
|
|
|
static inline int signal_pending_state(long state, struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
if (!(state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))
|
|
|
|
|
return 0;
|
|
|
|
|
if (!signal_pending(p))
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
static inline int need_resched(void)
|
|
|
|
|
{
|
2008-05-13 01:14:22 +08:00
|
|
|
return unlikely(test_thread_flag(TIF_NEED_RESCHED));
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* cond_resched() and cond_resched_lock(): latency reduction via
|
|
|
|
|
* explicit rescheduling in places that are safe. The return
|
|
|
|
|
* value indicates whether a reschedule was done in fact.
|
|
|
|
|
* cond_resched_lock() will drop the spinlock before scheduling,
|
|
|
|
|
* cond_resched_softirq() will enable bhs before scheduling.
|
|
|
|
|
*/
|
2008-05-12 07:04:48 +08:00
|
|
|
extern int _cond_resched(void);
|
2009-07-16 21:44:29 +08:00
|
|
|
|
2009-07-16 21:44:29 +08:00
|
|
|
#define cond_resched() ({ \
|
|
|
|
|
__might_sleep(__FILE__, __LINE__, 0); \
|
|
|
|
|
_cond_resched(); \
|
|
|
|
|
})
|
2009-07-16 21:44:29 +08:00
|
|
|
|
2009-07-16 21:44:29 +08:00
|
|
|
extern int __cond_resched_lock(spinlock_t *lock);
|
|
|
|
|
|
2009-07-25 02:05:23 +08:00
|
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
|
#define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET
|
2008-01-26 04:08:28 +08:00
|
|
|
#else
|
2009-07-25 02:05:23 +08:00
|
|
|
#define PREEMPT_LOCK_OFFSET 0
|
2008-01-26 04:08:28 +08:00
|
|
|
#endif
|
2009-07-25 02:05:23 +08:00
|
|
|
|
2009-07-16 21:44:29 +08:00
|
|
|
#define cond_resched_lock(lock) ({ \
|
2009-07-25 02:05:23 +08:00
|
|
|
__might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \
|
2009-07-16 21:44:29 +08:00
|
|
|
__cond_resched_lock(lock); \
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
extern int __cond_resched_softirq(void);
|
|
|
|
|
|
|
|
|
|
#define cond_resched_softirq() ({ \
|
|
|
|
|
__might_sleep(__FILE__, __LINE__, SOFTIRQ_OFFSET); \
|
|
|
|
|
__cond_resched_softirq(); \
|
|
|
|
|
})
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Does a critical section need to be broken due to another
|
2008-01-30 20:31:20 +08:00
|
|
|
* task waiting?: (technically does not depend on CONFIG_PREEMPT,
|
|
|
|
|
* but a general need for low latency)
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2008-01-30 20:31:20 +08:00
|
|
|
static inline int spin_needbreak(spinlock_t *lock)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2008-01-30 20:31:20 +08:00
|
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
|
return spin_is_contended(lock);
|
|
|
|
|
#else
|
2005-04-17 06:20:36 +08:00
|
|
|
return 0;
|
2008-01-30 20:31:20 +08:00
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
/*
|
|
|
|
|
* Thread group CPU time accounting.
|
|
|
|
|
*/
|
2009-02-05 19:24:16 +08:00
|
|
|
void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
|
2009-02-11 18:30:27 +08:00
|
|
|
void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
|
2008-11-25 00:06:57 +08:00
|
|
|
static inline void thread_group_cputime_init(struct signal_struct *sig)
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
{
|
2009-02-05 19:24:16 +08:00
|
|
|
spin_lock_init(&sig->cputimer.lock);
|
timers: fix itimer/many thread hang
Overview
This patch reworks the handling of POSIX CPU timers, including the
ITIMER_PROF, ITIMER_VIRT timers and rlimit handling. It was put together
with the help of Roland McGrath, the owner and original writer of this code.
The problem we ran into, and the reason for this rework, has to do with using
a profiling timer in a process with a large number of threads. It appears
that the performance of the old implementation of run_posix_cpu_timers() was
at least O(n*3) (where "n" is the number of threads in a process) or worse.
Everything is fine with an increasing number of threads until the time taken
for that routine to run becomes the same as or greater than the tick time, at
which point things degrade rather quickly.
This patch fixes bug 9906, "Weird hang with NPTL and SIGPROF."
Code Changes
This rework corrects the implementation of run_posix_cpu_timers() to make it
run in constant time for a particular machine. (Performance may vary between
one machine and another depending upon whether the kernel is built as single-
or multiprocessor and, in the latter case, depending upon the number of
running processors.) To do this, at each tick we now update fields in
signal_struct as well as task_struct. The run_posix_cpu_timers() function
uses those fields to make its decisions.
We define a new structure, "task_cputime," to contain user, system and
scheduler times and use these in appropriate places:
struct task_cputime {
cputime_t utime;
cputime_t stime;
unsigned long long sum_exec_runtime;
};
This is included in the structure "thread_group_cputime," which is a new
substructure of signal_struct and which varies for uniprocessor versus
multiprocessor kernels. For uniprocessor kernels, it uses "task_cputime" as
a simple substructure, while for multiprocessor kernels it is a pointer:
struct thread_group_cputime {
struct task_cputime totals;
};
struct thread_group_cputime {
struct task_cputime *totals;
};
We also add a new task_cputime substructure directly to signal_struct, to
cache the earliest expiration of process-wide timers, and task_cputime also
replaces the it_*_expires fields of task_struct (used for earliest expiration
of thread timers). The "thread_group_cputime" structure contains process-wide
timers that are updated via account_user_time() and friends. In the non-SMP
case the structure is a simple aggregator; unfortunately in the SMP case that
simplicity was not achievable due to cache-line contention between CPUs (in
one measured case performance was actually _worse_ on a 16-cpu system than
the same test on a 4-cpu system, due to this contention). For SMP, the
thread_group_cputime counters are maintained as a per-cpu structure allocated
using alloc_percpu(). The timer functions update only the timer field in
the structure corresponding to the running CPU, obtained using per_cpu_ptr().
We define a set of inline functions in sched.h that we use to maintain the
thread_group_cputime structure and hide the differences between UP and SMP
implementations from the rest of the kernel. The thread_group_cputime_init()
function initializes the thread_group_cputime structure for the given task.
The thread_group_cputime_alloc() is a no-op for UP; for SMP it calls the
out-of-line function thread_group_cputime_alloc_smp() to allocate and fill
in the per-cpu structures and fields. The thread_group_cputime_free()
function, also a no-op for UP, in SMP frees the per-cpu structures. The
thread_group_cputime_clone_thread() function (also a UP no-op) for SMP calls
thread_group_cputime_alloc() if the per-cpu structures haven't yet been
allocated. The thread_group_cputime() function fills the task_cputime
structure it is passed with the contents of the thread_group_cputime fields;
in UP it's that simple but in SMP it must also safely check that tsk->signal
is non-NULL (if it is it just uses the appropriate fields of task_struct) and,
if so, sums the per-cpu values for each online CPU. Finally, the three
functions account_group_user_time(), account_group_system_time() and
account_group_exec_runtime() are used by timer functions to update the
respective fields of the thread_group_cputime structure.
Non-SMP operation is trivial and will not be mentioned further.
The per-cpu structure is always allocated when a task creates its first new
thread, via a call to thread_group_cputime_clone_thread() from copy_signal().
It is freed at process exit via a call to thread_group_cputime_free() from
cleanup_signal().
All functions that formerly summed utime/stime/sum_sched_runtime values from
from all threads in the thread group now use thread_group_cputime() to
snapshot the values in the thread_group_cputime structure or the values in
the task structure itself if the per-cpu structure hasn't been allocated.
Finally, the code in kernel/posix-cpu-timers.c has changed quite a bit.
The run_posix_cpu_timers() function has been split into a fast path and a
slow path; the former safely checks whether there are any expired thread
timers and, if not, just returns, while the slow path does the heavy lifting.
With the dedicated thread group fields, timers are no longer "rebalanced" and
the process_timer_rebalance() function and related code has gone away. All
summing loops are gone and all code that used them now uses the
thread_group_cputime() inline. When process-wide timers are set, the new
task_cputime structure in signal_struct is used to cache the earliest
expiration; this is checked in the fast path.
Performance
The fix appears not to add significant overhead to existing operations. It
generally performs the same as the current code except in two cases, one in
which it performs slightly worse (Case 5 below) and one in which it performs
very significantly better (Case 2 below). Overall it's a wash except in those
two cases.
I've since done somewhat more involved testing on a dual-core Opteron system.
Case 1: With no itimer running, for a test with 100,000 threads, the fixed
kernel took 1428.5 seconds, 513 seconds more than the unfixed system,
all of which was spent in the system. There were twice as many
voluntary context switches with the fix as without it.
Case 2: With an itimer running at .01 second ticks and 4000 threads (the most
an unmodified kernel can handle), the fixed kernel ran the test in
eight percent of the time (5.8 seconds as opposed to 70 seconds) and
had better tick accuracy (.012 seconds per tick as opposed to .023
seconds per tick).
Case 3: A 4000-thread test with an initial timer tick of .01 second and an
interval of 10,000 seconds (i.e. a timer that ticks only once) had
very nearly the same performance in both cases: 6.3 seconds elapsed
for the fixed kernel versus 5.5 seconds for the unfixed kernel.
With fewer threads (eight in these tests), the Case 1 test ran in essentially
the same time on both the modified and unmodified kernels (5.2 seconds versus
5.8 seconds). The Case 2 test ran in about the same time as well, 5.9 seconds
versus 5.4 seconds but again with much better tick accuracy, .013 seconds per
tick versus .025 seconds per tick for the unmodified kernel.
Since the fix affected the rlimit code, I also tested soft and hard CPU limits.
Case 4: With a hard CPU limit of 20 seconds and eight threads (and an itimer
running), the modified kernel was very slightly favored in that while
it killed the process in 19.997 seconds of CPU time (5.002 seconds of
wall time), only .003 seconds of that was system time, the rest was
user time. The unmodified kernel killed the process in 20.001 seconds
of CPU (5.014 seconds of wall time) of which .016 seconds was system
time. Really, though, the results were too close to call. The results
were essentially the same with no itimer running.
Case 5: With a soft limit of 20 seconds and a hard limit of 2000 seconds
(where the hard limit would never be reached) and an itimer running,
the modified kernel exhibited worse tick accuracy than the unmodified
kernel: .050 seconds/tick versus .028 seconds/tick. Otherwise,
performance was almost indistinguishable. With no itimer running this
test exhibited virtually identical behavior and times in both cases.
In times past I did some limited performance testing. those results are below.
On a four-cpu Opteron system without this fix, a sixteen-thread test executed
in 3569.991 seconds, of which user was 3568.435s and system was 1.556s. On
the same system with the fix, user and elapsed time were about the same, but
system time dropped to 0.007 seconds. Performance with eight, four and one
thread were comparable. Interestingly, the timer ticks with the fix seemed
more accurate: The sixteen-thread test with the fix received 149543 ticks
for 0.024 seconds per tick, while the same test without the fix received 58720
for 0.061 seconds per tick. Both cases were configured for an interval of
0.01 seconds. Again, the other tests were comparable. Each thread in this
test computed the primes up to 25,000,000.
I also did a test with a large number of threads, 100,000 threads, which is
impossible without the fix. In this case each thread computed the primes only
up to 10,000 (to make the runtime manageable). System time dominated, at
1546.968 seconds out of a total 2176.906 seconds (giving a user time of
629.938s). It received 147651 ticks for 0.015 seconds per tick, still quite
accurate. There is obviously no comparable test without the fix.
Signed-off-by: Frank Mayhar <fmayhar@google.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-09-13 00:54:39 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void thread_group_cputime_free(struct signal_struct *sig)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
2007-05-24 04:57:44 +08:00
|
|
|
/*
|
|
|
|
|
* Reevaluate whether the task has signals pending delivery.
|
|
|
|
|
* Wake the task if so.
|
|
|
|
|
* This is required every time the blocked sigset_t changes.
|
|
|
|
|
* callers must hold sighand->siglock.
|
|
|
|
|
*/
|
|
|
|
|
extern void recalc_sigpending_and_wake(struct task_struct *t);
|
2005-04-17 06:20:36 +08:00
|
|
|
extern void recalc_sigpending(void);
|
|
|
|
|
|
|
|
|
|
extern void signal_wake_up(struct task_struct *t, int resume_stopped);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Wrappers for p->thread_info->cpu access. No-op on UP.
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
|
|
|
|
|
|
static inline unsigned int task_cpu(const struct task_struct *p)
|
|
|
|
|
{
|
2005-11-14 08:06:55 +08:00
|
|
|
return task_thread_info(p)->cpu;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2007-07-10 00:51:58 +08:00
|
|
|
extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
|
|
|
|
|
static inline unsigned int task_cpu(const struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
|
2008-05-13 03:20:52 +08:00
|
|
|
#ifdef CONFIG_TRACING
|
|
|
|
|
extern void
|
|
|
|
|
__trace_special(void *__tr, void *__data,
|
|
|
|
|
unsigned long arg1, unsigned long arg2, unsigned long arg3);
|
2005-04-17 06:20:36 +08:00
|
|
|
#else
|
2008-05-13 03:20:52 +08:00
|
|
|
static inline void
|
|
|
|
|
__trace_special(void *__tr, void *__data,
|
|
|
|
|
unsigned long arg1, unsigned long arg2, unsigned long arg3)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-11-25 00:05:14 +08:00
|
|
|
extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
|
|
|
|
|
extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
|
2006-06-27 17:54:42 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
extern void normalize_rt_tasks(void);
|
|
|
|
|
|
2010-01-20 20:26:18 +08:00
|
|
|
#ifdef CONFIG_CGROUP_SCHED
|
2007-10-15 23:00:09 +08:00
|
|
|
|
2007-10-15 23:00:14 +08:00
|
|
|
extern struct task_group init_task_group;
|
2007-10-15 23:00:09 +08:00
|
|
|
|
2008-04-20 01:44:59 +08:00
|
|
|
extern struct task_group *sched_create_group(struct task_group *parent);
|
2007-10-15 23:00:14 +08:00
|
|
|
extern void sched_destroy_group(struct task_group *tg);
|
2007-10-15 23:00:09 +08:00
|
|
|
extern void sched_move_task(struct task_struct *tsk);
|
2008-02-13 22:45:40 +08:00
|
|
|
#ifdef CONFIG_FAIR_GROUP_SCHED
|
2007-10-15 23:00:14 +08:00
|
|
|
extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);
|
2007-10-15 23:00:14 +08:00
|
|
|
extern unsigned long sched_group_shares(struct task_group *tg);
|
2008-02-13 22:45:40 +08:00
|
|
|
#endif
|
|
|
|
|
#ifdef CONFIG_RT_GROUP_SCHED
|
2008-02-13 22:45:39 +08:00
|
|
|
extern int sched_group_set_rt_runtime(struct task_group *tg,
|
|
|
|
|
long rt_runtime_us);
|
|
|
|
|
extern long sched_group_rt_runtime(struct task_group *tg);
|
2008-04-20 01:44:57 +08:00
|
|
|
extern int sched_group_set_rt_period(struct task_group *tg,
|
|
|
|
|
long rt_period_us);
|
|
|
|
|
extern long sched_group_rt_period(struct task_group *tg);
|
2009-02-27 17:43:54 +08:00
|
|
|
extern int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk);
|
2008-02-13 22:45:40 +08:00
|
|
|
#endif
|
2007-10-15 23:00:09 +08:00
|
|
|
#endif
|
|
|
|
|
|
2009-02-27 17:43:54 +08:00
|
|
|
extern int task_can_switch_user(struct user_struct *up,
|
|
|
|
|
struct task_struct *tsk);
|
|
|
|
|
|
[PATCH] ifdef ->rchar, ->wchar, ->syscr, ->syscw from task_struct
They are fat: 4x8 bytes in task_struct.
They are uncoditionally updated in every fork, read, write and sendfile.
They are used only if you have some "extended acct fields feature".
And please, please, please, read(2) knows about bytes, not characters,
why it is called "rchar"?
Signed-off-by: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Jay Lan <jlan@engr.sgi.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-10 17:46:45 +08:00
|
|
|
#ifdef CONFIG_TASK_XACCT
|
|
|
|
|
static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
|
|
|
|
|
{
|
2008-07-28 06:48:12 +08:00
|
|
|
tsk->ioac.rchar += amt;
|
[PATCH] ifdef ->rchar, ->wchar, ->syscr, ->syscw from task_struct
They are fat: 4x8 bytes in task_struct.
They are uncoditionally updated in every fork, read, write and sendfile.
They are used only if you have some "extended acct fields feature".
And please, please, please, read(2) knows about bytes, not characters,
why it is called "rchar"?
Signed-off-by: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Jay Lan <jlan@engr.sgi.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-10 17:46:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
|
|
|
|
|
{
|
2008-07-28 06:48:12 +08:00
|
|
|
tsk->ioac.wchar += amt;
|
[PATCH] ifdef ->rchar, ->wchar, ->syscr, ->syscw from task_struct
They are fat: 4x8 bytes in task_struct.
They are uncoditionally updated in every fork, read, write and sendfile.
They are used only if you have some "extended acct fields feature".
And please, please, please, read(2) knows about bytes, not characters,
why it is called "rchar"?
Signed-off-by: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Jay Lan <jlan@engr.sgi.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-10 17:46:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void inc_syscr(struct task_struct *tsk)
|
|
|
|
|
{
|
2008-07-28 06:48:12 +08:00
|
|
|
tsk->ioac.syscr++;
|
[PATCH] ifdef ->rchar, ->wchar, ->syscr, ->syscw from task_struct
They are fat: 4x8 bytes in task_struct.
They are uncoditionally updated in every fork, read, write and sendfile.
They are used only if you have some "extended acct fields feature".
And please, please, please, read(2) knows about bytes, not characters,
why it is called "rchar"?
Signed-off-by: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Jay Lan <jlan@engr.sgi.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-10 17:46:45 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void inc_syscw(struct task_struct *tsk)
|
|
|
|
|
{
|
2008-07-28 06:48:12 +08:00
|
|
|
tsk->ioac.syscw++;
|
[PATCH] ifdef ->rchar, ->wchar, ->syscr, ->syscw from task_struct
They are fat: 4x8 bytes in task_struct.
They are uncoditionally updated in every fork, read, write and sendfile.
They are used only if you have some "extended acct fields feature".
And please, please, please, read(2) knows about bytes, not characters,
why it is called "rchar"?
Signed-off-by: Alexey Dobriyan <adobriyan@gmail.com>
Cc: Jay Lan <jlan@engr.sgi.com>
Cc: Balbir Singh <balbir@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-10 17:46:45 +08:00
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
static inline void add_rchar(struct task_struct *tsk, ssize_t amt)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void add_wchar(struct task_struct *tsk, ssize_t amt)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void inc_syscr(struct task_struct *tsk)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void inc_syscw(struct task_struct *tsk)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-02-05 14:28:59 +08:00
|
|
|
#ifndef TASK_SIZE_OF
|
|
|
|
|
#define TASK_SIZE_OF(tsk) TASK_SIZE
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-12-05 03:12:29 +08:00
|
|
|
/*
|
|
|
|
|
* Call the function if the target task is executing on a CPU right now:
|
|
|
|
|
*/
|
|
|
|
|
extern void task_oncpu_function_call(struct task_struct *p,
|
|
|
|
|
void (*func) (void *info), void *info);
|
|
|
|
|
|
|
|
|
|
|
cgroups: add an owner to the mm_struct
Remove the mem_cgroup member from mm_struct and instead adds an owner.
This approach was suggested by Paul Menage. The advantage of this approach
is that, once the mm->owner is known, using the subsystem id, the cgroup
can be determined. It also allows several control groups that are
virtually grouped by mm_struct, to exist independent of the memory
controller i.e., without adding mem_cgroup's for each controller, to
mm_struct.
A new config option CONFIG_MM_OWNER is added and the memory resource
controller selects this config option.
This patch also adds cgroup callbacks to notify subsystems when mm->owner
changes. The mm_cgroup_changed callback is called with the task_lock() of
the new task held and is called just prior to changing the mm->owner.
I am indebted to Paul Menage for the several reviews of this patchset and
helping me make it lighter and simpler.
This patch was tested on a powerpc box, it was compiled with both the
MM_OWNER config turned on and off.
After the thread group leader exits, it's moved to init_css_state by
cgroup_exit(), thus all future charges from runnings threads would be
redirected to the init_css_set's subsystem.
Signed-off-by: Balbir Singh <balbir@linux.vnet.ibm.com>
Cc: Pavel Emelianov <xemul@openvz.org>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Sudhir Kumar <skumar@linux.vnet.ibm.com>
Cc: YAMAMOTO Takashi <yamamoto@valinux.co.jp>
Cc: Hirokazu Takahashi <taka@valinux.co.jp>
Cc: David Rientjes <rientjes@google.com>,
Cc: Balbir Singh <balbir@linux.vnet.ibm.com>
Acked-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>
Reviewed-by: Paul Menage <menage@google.com>
Cc: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-04-29 16:00:16 +08:00
|
|
|
#ifdef CONFIG_MM_OWNER
|
|
|
|
|
extern void mm_update_next_owner(struct mm_struct *mm);
|
|
|
|
|
extern void mm_init_owner(struct mm_struct *mm, struct task_struct *p);
|
|
|
|
|
#else
|
|
|
|
|
static inline void mm_update_next_owner(struct mm_struct *mm)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
#endif /* CONFIG_MM_OWNER */
|
|
|
|
|
|
2009-11-20 00:16:37 +08:00
|
|
|
static inline unsigned long task_rlimit(const struct task_struct *tsk,
|
|
|
|
|
unsigned int limit)
|
|
|
|
|
{
|
|
|
|
|
return ACCESS_ONCE(tsk->signal->rlim[limit].rlim_cur);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long task_rlimit_max(const struct task_struct *tsk,
|
|
|
|
|
unsigned int limit)
|
|
|
|
|
{
|
|
|
|
|
return ACCESS_ONCE(tsk->signal->rlim[limit].rlim_max);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long rlimit(unsigned int limit)
|
|
|
|
|
{
|
|
|
|
|
return task_rlimit(current, limit);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long rlimit_max(unsigned int limit)
|
|
|
|
|
{
|
|
|
|
|
return task_rlimit_max(current, limit);
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#endif /* __KERNEL__ */
|
|
|
|
|
|
|
|
|
|
#endif
|
