mirror of
https://github.com/edk2-porting/linux-next.git
synced 2024-12-16 01:04:08 +08:00
sched: Clean up scheduler_ipi()
The scheduler IPI has grown weird and wonderful over the years, time for spring cleaning. Move all the non-trivial stuff out of it and into a regular smp function call IPI. This then reduces the schedule_ipi() to most of it's former NOP glory and ensures to keep the interrupt vector lean and mean. Aside of that avoiding the full irq_enter() in the x86 IPI implementation is incorrect as scheduler_ipi() can be instrumented. To work around that scheduler_ipi() had an irq_enter/exit() hack when heavy work was pending. This is gone now. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com> Link: https://lkml.kernel.org/r/20200505134058.361859938@linutronix.de
This commit is contained in:
parent
b1d1779e5e
commit
90b5363acd
@ -219,6 +219,13 @@ void update_rq_clock(struct rq *rq)
|
||||
update_rq_clock_task(rq, delta);
|
||||
}
|
||||
|
||||
static inline void
|
||||
rq_csd_init(struct rq *rq, call_single_data_t *csd, smp_call_func_t func)
|
||||
{
|
||||
csd->flags = 0;
|
||||
csd->func = func;
|
||||
csd->info = rq;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SCHED_HRTICK
|
||||
/*
|
||||
@ -314,16 +321,14 @@ void hrtick_start(struct rq *rq, u64 delay)
|
||||
hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay),
|
||||
HRTIMER_MODE_REL_PINNED_HARD);
|
||||
}
|
||||
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
static void hrtick_rq_init(struct rq *rq)
|
||||
{
|
||||
#ifdef CONFIG_SMP
|
||||
rq->hrtick_csd.flags = 0;
|
||||
rq->hrtick_csd.func = __hrtick_start;
|
||||
rq->hrtick_csd.info = rq;
|
||||
rq_csd_init(rq, &rq->hrtick_csd, __hrtick_start);
|
||||
#endif
|
||||
|
||||
hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
|
||||
rq->hrtick_timer.function = hrtick;
|
||||
}
|
||||
@ -650,6 +655,16 @@ static inline bool got_nohz_idle_kick(void)
|
||||
return false;
|
||||
}
|
||||
|
||||
static void nohz_csd_func(void *info)
|
||||
{
|
||||
struct rq *rq = info;
|
||||
|
||||
if (got_nohz_idle_kick()) {
|
||||
rq->idle_balance = 1;
|
||||
raise_softirq_irqoff(SCHED_SOFTIRQ);
|
||||
}
|
||||
}
|
||||
|
||||
#else /* CONFIG_NO_HZ_COMMON */
|
||||
|
||||
static inline bool got_nohz_idle_kick(void)
|
||||
@ -2292,6 +2307,11 @@ void sched_ttwu_pending(void)
|
||||
rq_unlock_irqrestore(rq, &rf);
|
||||
}
|
||||
|
||||
static void wake_csd_func(void *info)
|
||||
{
|
||||
sched_ttwu_pending();
|
||||
}
|
||||
|
||||
void scheduler_ipi(void)
|
||||
{
|
||||
/*
|
||||
@ -2300,34 +2320,6 @@ void scheduler_ipi(void)
|
||||
* this IPI.
|
||||
*/
|
||||
preempt_fold_need_resched();
|
||||
|
||||
if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
|
||||
return;
|
||||
|
||||
/*
|
||||
* Not all reschedule IPI handlers call irq_enter/irq_exit, since
|
||||
* traditionally all their work was done from the interrupt return
|
||||
* path. Now that we actually do some work, we need to make sure
|
||||
* we do call them.
|
||||
*
|
||||
* Some archs already do call them, luckily irq_enter/exit nest
|
||||
* properly.
|
||||
*
|
||||
* Arguably we should visit all archs and update all handlers,
|
||||
* however a fair share of IPIs are still resched only so this would
|
||||
* somewhat pessimize the simple resched case.
|
||||
*/
|
||||
irq_enter();
|
||||
sched_ttwu_pending();
|
||||
|
||||
/*
|
||||
* Check if someone kicked us for doing the nohz idle load balance.
|
||||
*/
|
||||
if (unlikely(got_nohz_idle_kick())) {
|
||||
this_rq()->idle_balance = 1;
|
||||
raise_softirq_irqoff(SCHED_SOFTIRQ);
|
||||
}
|
||||
irq_exit();
|
||||
}
|
||||
|
||||
static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
|
||||
@ -2336,9 +2328,9 @@ static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
|
||||
|
||||
p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
|
||||
|
||||
if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
|
||||
if (llist_add(&p->wake_entry, &rq->wake_list)) {
|
||||
if (!set_nr_if_polling(rq->idle))
|
||||
smp_send_reschedule(cpu);
|
||||
smp_call_function_single_async(cpu, &rq->wake_csd);
|
||||
else
|
||||
trace_sched_wake_idle_without_ipi(cpu);
|
||||
}
|
||||
@ -6693,12 +6685,16 @@ void __init sched_init(void)
|
||||
rq->avg_idle = 2*sysctl_sched_migration_cost;
|
||||
rq->max_idle_balance_cost = sysctl_sched_migration_cost;
|
||||
|
||||
rq_csd_init(rq, &rq->wake_csd, wake_csd_func);
|
||||
|
||||
INIT_LIST_HEAD(&rq->cfs_tasks);
|
||||
|
||||
rq_attach_root(rq, &def_root_domain);
|
||||
#ifdef CONFIG_NO_HZ_COMMON
|
||||
rq->last_blocked_load_update_tick = jiffies;
|
||||
atomic_set(&rq->nohz_flags, 0);
|
||||
|
||||
rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func);
|
||||
#endif
|
||||
#endif /* CONFIG_SMP */
|
||||
hrtick_rq_init(rq);
|
||||
|
@ -10000,12 +10000,11 @@ static void kick_ilb(unsigned int flags)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Use smp_send_reschedule() instead of resched_cpu().
|
||||
* This way we generate a sched IPI on the target CPU which
|
||||
* This way we generate an IPI on the target CPU which
|
||||
* is idle. And the softirq performing nohz idle load balance
|
||||
* will be run before returning from the IPI.
|
||||
*/
|
||||
smp_send_reschedule(ilb_cpu);
|
||||
smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -889,9 +889,10 @@ struct rq {
|
||||
#ifdef CONFIG_SMP
|
||||
unsigned long last_blocked_load_update_tick;
|
||||
unsigned int has_blocked_load;
|
||||
call_single_data_t nohz_csd;
|
||||
#endif /* CONFIG_SMP */
|
||||
unsigned int nohz_tick_stopped;
|
||||
atomic_t nohz_flags;
|
||||
atomic_t nohz_flags;
|
||||
#endif /* CONFIG_NO_HZ_COMMON */
|
||||
|
||||
unsigned long nr_load_updates;
|
||||
@ -978,7 +979,7 @@ struct rq {
|
||||
|
||||
/* This is used to determine avg_idle's max value */
|
||||
u64 max_idle_balance_cost;
|
||||
#endif
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
|
||||
u64 prev_irq_time;
|
||||
@ -1020,6 +1021,7 @@ struct rq {
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
call_single_data_t wake_csd;
|
||||
struct llist_head wake_list;
|
||||
#endif
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user