mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-11-26 13:44:15 +08:00
6be9238e5c
Introduce four new variants of the async_schedule_ functions that allow scheduling on a specific NUMA node. The first two functions are async_schedule_near and async_schedule_near_domain end up mapping to async_schedule and async_schedule_domain, but provide NUMA node specific functionality. They replace the original functions which were moved to inline function definitions that call the new functions while passing NUMA_NO_NODE. The second two functions are async_schedule_dev and async_schedule_dev_domain which provide NUMA specific functionality when passing a device as the data member and that device has a NUMA node other than NUMA_NO_NODE. The main motivation behind this is to address the need to be able to schedule device specific init work on specific NUMA nodes in order to improve performance of memory initialization. I have seen a significant improvement in initialziation time for persistent memory as a result of this approach. In the case of 3TB of memory on a single node the initialization time in the worst case went from 36s down to about 26s for a 10s improvement. As such the data shows a general benefit for affinitizing the async work to the node local to the device. Reviewed-by: Bart Van Assche <bvanassche@acm.org> Reviewed-by: Dan Williams <dan.j.williams@intel.com> Signed-off-by: Alexander Duyck <alexander.h.duyck@linux.intel.com> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
127 lines
4.5 KiB
C
127 lines
4.5 KiB
C
/*
|
|
* async.h: Asynchronous function calls for boot performance
|
|
*
|
|
* (C) Copyright 2009 Intel Corporation
|
|
* Author: Arjan van de Ven <arjan@linux.intel.com>
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; version 2
|
|
* of the License.
|
|
*/
|
|
#ifndef __ASYNC_H__
|
|
#define __ASYNC_H__
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/list.h>
|
|
#include <linux/numa.h>
|
|
#include <linux/device.h>
|
|
|
|
typedef u64 async_cookie_t;
|
|
typedef void (*async_func_t) (void *data, async_cookie_t cookie);
|
|
struct async_domain {
|
|
struct list_head pending;
|
|
unsigned registered:1;
|
|
};
|
|
|
|
/*
|
|
* domain participates in global async_synchronize_full
|
|
*/
|
|
#define ASYNC_DOMAIN(_name) \
|
|
struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \
|
|
.registered = 1 }
|
|
|
|
/*
|
|
* domain is free to go out of scope as soon as all pending work is
|
|
* complete, this domain does not participate in async_synchronize_full
|
|
*/
|
|
#define ASYNC_DOMAIN_EXCLUSIVE(_name) \
|
|
struct async_domain _name = { .pending = LIST_HEAD_INIT(_name.pending), \
|
|
.registered = 0 }
|
|
|
|
async_cookie_t async_schedule_node(async_func_t func, void *data,
|
|
int node);
|
|
async_cookie_t async_schedule_node_domain(async_func_t func, void *data,
|
|
int node,
|
|
struct async_domain *domain);
|
|
|
|
/**
|
|
* async_schedule - schedule a function for asynchronous execution
|
|
* @func: function to execute asynchronously
|
|
* @data: data pointer to pass to the function
|
|
*
|
|
* Returns an async_cookie_t that may be used for checkpointing later.
|
|
* Note: This function may be called from atomic or non-atomic contexts.
|
|
*/
|
|
static inline async_cookie_t async_schedule(async_func_t func, void *data)
|
|
{
|
|
return async_schedule_node(func, data, NUMA_NO_NODE);
|
|
}
|
|
|
|
/**
|
|
* async_schedule_domain - schedule a function for asynchronous execution within a certain domain
|
|
* @func: function to execute asynchronously
|
|
* @data: data pointer to pass to the function
|
|
* @domain: the domain
|
|
*
|
|
* Returns an async_cookie_t that may be used for checkpointing later.
|
|
* @domain may be used in the async_synchronize_*_domain() functions to
|
|
* wait within a certain synchronization domain rather than globally.
|
|
* Note: This function may be called from atomic or non-atomic contexts.
|
|
*/
|
|
static inline async_cookie_t
|
|
async_schedule_domain(async_func_t func, void *data,
|
|
struct async_domain *domain)
|
|
{
|
|
return async_schedule_node_domain(func, data, NUMA_NO_NODE, domain);
|
|
}
|
|
|
|
/**
|
|
* async_schedule_dev - A device specific version of async_schedule
|
|
* @func: function to execute asynchronously
|
|
* @dev: device argument to be passed to function
|
|
*
|
|
* Returns an async_cookie_t that may be used for checkpointing later.
|
|
* @dev is used as both the argument for the function and to provide NUMA
|
|
* context for where to run the function. By doing this we can try to
|
|
* provide for the best possible outcome by operating on the device on the
|
|
* CPUs closest to the device.
|
|
* Note: This function may be called from atomic or non-atomic contexts.
|
|
*/
|
|
static inline async_cookie_t
|
|
async_schedule_dev(async_func_t func, struct device *dev)
|
|
{
|
|
return async_schedule_node(func, dev, dev_to_node(dev));
|
|
}
|
|
|
|
/**
|
|
* async_schedule_dev_domain - A device specific version of async_schedule_domain
|
|
* @func: function to execute asynchronously
|
|
* @dev: device argument to be passed to function
|
|
* @domain: the domain
|
|
*
|
|
* Returns an async_cookie_t that may be used for checkpointing later.
|
|
* @dev is used as both the argument for the function and to provide NUMA
|
|
* context for where to run the function. By doing this we can try to
|
|
* provide for the best possible outcome by operating on the device on the
|
|
* CPUs closest to the device.
|
|
* @domain may be used in the async_synchronize_*_domain() functions to
|
|
* wait within a certain synchronization domain rather than globally.
|
|
* Note: This function may be called from atomic or non-atomic contexts.
|
|
*/
|
|
static inline async_cookie_t
|
|
async_schedule_dev_domain(async_func_t func, struct device *dev,
|
|
struct async_domain *domain)
|
|
{
|
|
return async_schedule_node_domain(func, dev, dev_to_node(dev), domain);
|
|
}
|
|
|
|
void async_unregister_domain(struct async_domain *domain);
|
|
extern void async_synchronize_full(void);
|
|
extern void async_synchronize_full_domain(struct async_domain *domain);
|
|
extern void async_synchronize_cookie(async_cookie_t cookie);
|
|
extern void async_synchronize_cookie_domain(async_cookie_t cookie,
|
|
struct async_domain *domain);
|
|
extern bool current_is_async(void);
|
|
#endif
|