mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-12 13:34:10 +08:00
97fd6016a7
Currently numa_clear_node removes both cpu information from the NUMA node cpumap as well as the NUMA node id from the cpu. Similarly numa_store_cpu_info updates both percpu nodeid and NUMA cpumap. However we need to retain the numa node id for the cpu and only remove the cpu information from the numa node cpumap during CPU hotplug out. The same can be extended for hotplugging in the CPU. This patch separates out numa_{add,remove}_cpu from numa_clear_node and numa_store_cpu_info. Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Will Deacon <will.deacon@arm.com> Reviewed-by: Ganapatrao Kulkarni <ganapatrao.kulkarni@cavium.com> Tested-by: Ganapatrao Kulkarni <ganapatrao.kulkarni@cavium.com> Tested-by: Hanjun Guo <hanjun.guo@linaro.org> Signed-off-by: Sudeep Holla <sudeep.holla@arm.com> Signed-off-by: Will Deacon <will.deacon@arm.com>
467 lines
11 KiB
C
467 lines
11 KiB
C
/*
|
|
* NUMA support, based on the x86 implementation.
|
|
*
|
|
* Copyright (C) 2015 Cavium Inc.
|
|
* Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#define pr_fmt(fmt) "NUMA: " fmt
|
|
|
|
#include <linux/acpi.h>
|
|
#include <linux/bootmem.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/module.h>
|
|
#include <linux/of.h>
|
|
|
|
#include <asm/acpi.h>
|
|
#include <asm/sections.h>
|
|
|
|
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
|
|
EXPORT_SYMBOL(node_data);
|
|
nodemask_t numa_nodes_parsed __initdata;
|
|
static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };
|
|
|
|
static int numa_distance_cnt;
|
|
static u8 *numa_distance;
|
|
bool numa_off;
|
|
|
|
static __init int numa_parse_early_param(char *opt)
|
|
{
|
|
if (!opt)
|
|
return -EINVAL;
|
|
if (!strncmp(opt, "off", 3))
|
|
numa_off = true;
|
|
|
|
return 0;
|
|
}
|
|
early_param("numa", numa_parse_early_param);
|
|
|
|
cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
|
|
EXPORT_SYMBOL(node_to_cpumask_map);
|
|
|
|
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
|
|
|
/*
|
|
* Returns a pointer to the bitmask of CPUs on Node 'node'.
|
|
*/
|
|
const struct cpumask *cpumask_of_node(int node)
|
|
{
|
|
if (WARN_ON(node >= nr_node_ids))
|
|
return cpu_none_mask;
|
|
|
|
if (WARN_ON(node_to_cpumask_map[node] == NULL))
|
|
return cpu_online_mask;
|
|
|
|
return node_to_cpumask_map[node];
|
|
}
|
|
EXPORT_SYMBOL(cpumask_of_node);
|
|
|
|
#endif
|
|
|
|
static void numa_update_cpu(unsigned int cpu, bool remove)
|
|
{
|
|
int nid = cpu_to_node(cpu);
|
|
|
|
if (nid == NUMA_NO_NODE)
|
|
return;
|
|
|
|
if (remove)
|
|
cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]);
|
|
else
|
|
cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
|
|
}
|
|
|
|
void numa_add_cpu(unsigned int cpu)
|
|
{
|
|
numa_update_cpu(cpu, false);
|
|
}
|
|
|
|
void numa_remove_cpu(unsigned int cpu)
|
|
{
|
|
numa_update_cpu(cpu, true);
|
|
}
|
|
|
|
void numa_clear_node(unsigned int cpu)
|
|
{
|
|
numa_remove_cpu(cpu);
|
|
set_cpu_numa_node(cpu, NUMA_NO_NODE);
|
|
}
|
|
|
|
/*
|
|
* Allocate node_to_cpumask_map based on number of available nodes
|
|
* Requires node_possible_map to be valid.
|
|
*
|
|
* Note: cpumask_of_node() is not valid until after this is done.
|
|
* (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
|
|
*/
|
|
static void __init setup_node_to_cpumask_map(void)
|
|
{
|
|
int node;
|
|
|
|
/* setup nr_node_ids if not done yet */
|
|
if (nr_node_ids == MAX_NUMNODES)
|
|
setup_nr_node_ids();
|
|
|
|
/* allocate and clear the mapping */
|
|
for (node = 0; node < nr_node_ids; node++) {
|
|
alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
|
|
cpumask_clear(node_to_cpumask_map[node]);
|
|
}
|
|
|
|
/* cpumask_of_node() will now work */
|
|
pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids);
|
|
}
|
|
|
|
/*
|
|
* Set the cpu to node and mem mapping
|
|
*/
|
|
void numa_store_cpu_info(unsigned int cpu)
|
|
{
|
|
set_cpu_numa_node(cpu, cpu_to_node_map[cpu]);
|
|
}
|
|
|
|
void __init early_map_cpu_to_node(unsigned int cpu, int nid)
|
|
{
|
|
/* fallback to node 0 */
|
|
if (nid < 0 || nid >= MAX_NUMNODES || numa_off)
|
|
nid = 0;
|
|
|
|
cpu_to_node_map[cpu] = nid;
|
|
|
|
/*
|
|
* We should set the numa node of cpu0 as soon as possible, because it
|
|
* has already been set up online before. cpu_to_node(0) will soon be
|
|
* called.
|
|
*/
|
|
if (!cpu)
|
|
set_cpu_numa_node(cpu, nid);
|
|
}
|
|
|
|
#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
|
|
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
|
|
EXPORT_SYMBOL(__per_cpu_offset);
|
|
|
|
static int __init early_cpu_to_node(int cpu)
|
|
{
|
|
return cpu_to_node_map[cpu];
|
|
}
|
|
|
|
static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
|
|
{
|
|
return node_distance(early_cpu_to_node(from), early_cpu_to_node(to));
|
|
}
|
|
|
|
static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size,
|
|
size_t align)
|
|
{
|
|
int nid = early_cpu_to_node(cpu);
|
|
|
|
return memblock_virt_alloc_try_nid(size, align,
|
|
__pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, nid);
|
|
}
|
|
|
|
static void __init pcpu_fc_free(void *ptr, size_t size)
|
|
{
|
|
memblock_free_early(__pa(ptr), size);
|
|
}
|
|
|
|
void __init setup_per_cpu_areas(void)
|
|
{
|
|
unsigned long delta;
|
|
unsigned int cpu;
|
|
int rc;
|
|
|
|
/*
|
|
* Always reserve area for module percpu variables. That's
|
|
* what the legacy allocator did.
|
|
*/
|
|
rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
|
|
PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
|
|
pcpu_cpu_distance,
|
|
pcpu_fc_alloc, pcpu_fc_free);
|
|
if (rc < 0)
|
|
panic("Failed to initialize percpu areas.");
|
|
|
|
delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
|
|
for_each_possible_cpu(cpu)
|
|
__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
|
|
}
|
|
#endif
|
|
|
|
/**
|
|
* numa_add_memblk - Set node id to memblk
|
|
* @nid: NUMA node ID of the new memblk
|
|
* @start: Start address of the new memblk
|
|
* @end: End address of the new memblk
|
|
*
|
|
* RETURNS:
|
|
* 0 on success, -errno on failure.
|
|
*/
|
|
int __init numa_add_memblk(int nid, u64 start, u64 end)
|
|
{
|
|
int ret;
|
|
|
|
ret = memblock_set_node(start, (end - start), &memblock.memory, nid);
|
|
if (ret < 0) {
|
|
pr_err("memblock [0x%llx - 0x%llx] failed to add on node %d\n",
|
|
start, (end - 1), nid);
|
|
return ret;
|
|
}
|
|
|
|
node_set(nid, numa_nodes_parsed);
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* Initialize NODE_DATA for a node on the local memory
|
|
*/
|
|
static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn)
|
|
{
|
|
const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES);
|
|
u64 nd_pa;
|
|
void *nd;
|
|
int tnid;
|
|
|
|
if (start_pfn >= end_pfn)
|
|
pr_info("Initmem setup node %d [<memory-less node>]\n", nid);
|
|
|
|
nd_pa = memblock_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid);
|
|
nd = __va(nd_pa);
|
|
|
|
/* report and initialize */
|
|
pr_info("NODE_DATA [mem %#010Lx-%#010Lx]\n",
|
|
nd_pa, nd_pa + nd_size - 1);
|
|
tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
|
|
if (tnid != nid)
|
|
pr_info("NODE_DATA(%d) on node %d\n", nid, tnid);
|
|
|
|
node_data[nid] = nd;
|
|
memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
|
|
NODE_DATA(nid)->node_id = nid;
|
|
NODE_DATA(nid)->node_start_pfn = start_pfn;
|
|
NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
|
|
}
|
|
|
|
/**
|
|
* numa_free_distance
|
|
*
|
|
* The current table is freed.
|
|
*/
|
|
void __init numa_free_distance(void)
|
|
{
|
|
size_t size;
|
|
|
|
if (!numa_distance)
|
|
return;
|
|
|
|
size = numa_distance_cnt * numa_distance_cnt *
|
|
sizeof(numa_distance[0]);
|
|
|
|
memblock_free(__pa(numa_distance), size);
|
|
numa_distance_cnt = 0;
|
|
numa_distance = NULL;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* Create a new NUMA distance table.
|
|
*
|
|
*/
|
|
static int __init numa_alloc_distance(void)
|
|
{
|
|
size_t size;
|
|
u64 phys;
|
|
int i, j;
|
|
|
|
size = nr_node_ids * nr_node_ids * sizeof(numa_distance[0]);
|
|
phys = memblock_find_in_range(0, PFN_PHYS(max_pfn),
|
|
size, PAGE_SIZE);
|
|
if (WARN_ON(!phys))
|
|
return -ENOMEM;
|
|
|
|
memblock_reserve(phys, size);
|
|
|
|
numa_distance = __va(phys);
|
|
numa_distance_cnt = nr_node_ids;
|
|
|
|
/* fill with the default distances */
|
|
for (i = 0; i < numa_distance_cnt; i++)
|
|
for (j = 0; j < numa_distance_cnt; j++)
|
|
numa_distance[i * numa_distance_cnt + j] = i == j ?
|
|
LOCAL_DISTANCE : REMOTE_DISTANCE;
|
|
|
|
pr_debug("Initialized distance table, cnt=%d\n", numa_distance_cnt);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* numa_set_distance - Set inter node NUMA distance from node to node.
|
|
* @from: the 'from' node to set distance
|
|
* @to: the 'to' node to set distance
|
|
* @distance: NUMA distance
|
|
*
|
|
* Set the distance from node @from to @to to @distance.
|
|
* If distance table doesn't exist, a warning is printed.
|
|
*
|
|
* If @from or @to is higher than the highest known node or lower than zero
|
|
* or @distance doesn't make sense, the call is ignored.
|
|
*
|
|
*/
|
|
void __init numa_set_distance(int from, int to, int distance)
|
|
{
|
|
if (!numa_distance) {
|
|
pr_warn_once("Warning: distance table not allocated yet\n");
|
|
return;
|
|
}
|
|
|
|
if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
|
|
from < 0 || to < 0) {
|
|
pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
|
|
from, to, distance);
|
|
return;
|
|
}
|
|
|
|
if ((u8)distance != distance ||
|
|
(from == to && distance != LOCAL_DISTANCE)) {
|
|
pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
|
|
from, to, distance);
|
|
return;
|
|
}
|
|
|
|
numa_distance[from * numa_distance_cnt + to] = distance;
|
|
}
|
|
|
|
/**
|
|
* Return NUMA distance @from to @to
|
|
*/
|
|
int __node_distance(int from, int to)
|
|
{
|
|
if (from >= numa_distance_cnt || to >= numa_distance_cnt)
|
|
return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
|
|
return numa_distance[from * numa_distance_cnt + to];
|
|
}
|
|
EXPORT_SYMBOL(__node_distance);
|
|
|
|
static int __init numa_register_nodes(void)
|
|
{
|
|
int nid;
|
|
struct memblock_region *mblk;
|
|
|
|
/* Check that valid nid is set to memblks */
|
|
for_each_memblock(memory, mblk)
|
|
if (mblk->nid == NUMA_NO_NODE || mblk->nid >= MAX_NUMNODES) {
|
|
pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
|
|
mblk->nid, mblk->base,
|
|
mblk->base + mblk->size - 1);
|
|
return -EINVAL;
|
|
}
|
|
|
|
/* Finally register nodes. */
|
|
for_each_node_mask(nid, numa_nodes_parsed) {
|
|
unsigned long start_pfn, end_pfn;
|
|
|
|
get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
|
|
setup_node_data(nid, start_pfn, end_pfn);
|
|
node_set_online(nid);
|
|
}
|
|
|
|
/* Setup online nodes to actual nodes*/
|
|
node_possible_map = numa_nodes_parsed;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int __init numa_init(int (*init_func)(void))
|
|
{
|
|
int ret;
|
|
|
|
nodes_clear(numa_nodes_parsed);
|
|
nodes_clear(node_possible_map);
|
|
nodes_clear(node_online_map);
|
|
numa_free_distance();
|
|
|
|
ret = numa_alloc_distance();
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
ret = init_func();
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
if (nodes_empty(numa_nodes_parsed)) {
|
|
pr_info("No NUMA configuration found\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
ret = numa_register_nodes();
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
setup_node_to_cpumask_map();
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* dummy_numa_init - Fallback dummy NUMA init
|
|
*
|
|
* Used if there's no underlying NUMA architecture, NUMA initialization
|
|
* fails, or NUMA is disabled on the command line.
|
|
*
|
|
* Must online at least one node (node 0) and add memory blocks that cover all
|
|
* allowed memory. It is unlikely that this function fails.
|
|
*/
|
|
static int __init dummy_numa_init(void)
|
|
{
|
|
int ret;
|
|
struct memblock_region *mblk;
|
|
|
|
if (numa_off)
|
|
pr_info("NUMA disabled\n"); /* Forced off on command line. */
|
|
pr_info("Faking a node at [mem %#018Lx-%#018Lx]\n",
|
|
0LLU, PFN_PHYS(max_pfn) - 1);
|
|
|
|
for_each_memblock(memory, mblk) {
|
|
ret = numa_add_memblk(0, mblk->base, mblk->base + mblk->size);
|
|
if (!ret)
|
|
continue;
|
|
|
|
pr_err("NUMA init failed\n");
|
|
return ret;
|
|
}
|
|
|
|
numa_off = true;
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* arm64_numa_init - Initialize NUMA
|
|
*
|
|
* Try each configured NUMA initialization method until one succeeds. The
|
|
* last fallback is dummy single node config encomapssing whole memory.
|
|
*/
|
|
void __init arm64_numa_init(void)
|
|
{
|
|
if (!numa_off) {
|
|
if (!acpi_disabled && !numa_init(arm64_acpi_numa_init))
|
|
return;
|
|
if (acpi_disabled && !numa_init(of_numa_init))
|
|
return;
|
|
}
|
|
|
|
numa_init(dummy_numa_init);
|
|
}
|