2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* PowerPC64 port by Mike Corrigan and Dave Engebretsen
|
|
|
|
* {mikejc|engebret}@us.ibm.com
|
|
|
|
*
|
|
|
|
* Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com>
|
|
|
|
*
|
|
|
|
* SMP scalability work:
|
|
|
|
* Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
|
|
|
|
*
|
|
|
|
* Module name: htab.c
|
|
|
|
*
|
|
|
|
* Description:
|
|
|
|
* PowerPC Hashed Page Table functions
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#undef DEBUG
|
2005-11-07 08:06:55 +08:00
|
|
|
#undef DEBUG_LOW
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/proc_fs.h>
|
|
|
|
#include <linux/stat.h>
|
|
|
|
#include <linux/sysctl.h>
|
2011-05-27 22:46:24 +08:00
|
|
|
#include <linux/export.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/ctype.h>
|
|
|
|
#include <linux/cache.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/signal.h>
|
2010-07-12 12:36:09 +08:00
|
|
|
#include <linux/memblock.h>
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
#include <linux/context_tracking.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
#include <asm/processor.h>
|
|
|
|
#include <asm/pgtable.h>
|
|
|
|
#include <asm/mmu.h>
|
|
|
|
#include <asm/mmu_context.h>
|
|
|
|
#include <asm/page.h>
|
|
|
|
#include <asm/types.h>
|
|
|
|
#include <asm/uaccess.h>
|
|
|
|
#include <asm/machdep.h>
|
2008-02-14 08:56:49 +08:00
|
|
|
#include <asm/prom.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
#include <asm/io.h>
|
|
|
|
#include <asm/eeh.h>
|
|
|
|
#include <asm/tlb.h>
|
|
|
|
#include <asm/cacheflush.h>
|
|
|
|
#include <asm/cputable.h>
|
|
|
|
#include <asm/sections.h>
|
2007-05-08 14:27:27 +08:00
|
|
|
#include <asm/spu.h>
|
2007-10-30 03:24:19 +08:00
|
|
|
#include <asm/udbg.h>
|
2011-04-05 07:56:18 +08:00
|
|
|
#include <asm/code-patching.h>
|
2012-02-20 10:15:03 +08:00
|
|
|
#include <asm/fadump.h>
|
2012-03-16 02:18:00 +08:00
|
|
|
#include <asm/firmware.h>
|
2013-02-14 00:21:40 +08:00
|
|
|
#include <asm/tm.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
#ifdef DEBUG
|
|
|
|
#define DBG(fmt...) udbg_printf(fmt)
|
|
|
|
#else
|
|
|
|
#define DBG(fmt...)
|
|
|
|
#endif
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
#ifdef DEBUG_LOW
|
|
|
|
#define DBG_LOW(fmt...) udbg_printf(fmt)
|
|
|
|
#else
|
|
|
|
#define DBG_LOW(fmt...)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define KB (1024)
|
|
|
|
#define MB (1024*KB)
|
2008-07-24 12:27:54 +08:00
|
|
|
#define GB (1024L*MB)
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Note: pte --> Linux PTE
|
|
|
|
* HPTE --> PowerPC Hashed Page Table Entry
|
|
|
|
*
|
|
|
|
* Execution context:
|
|
|
|
* htab_initialize is called with the MMU off (of course), but
|
|
|
|
* the kernel has been copied down to zero so it can directly
|
|
|
|
* reference global data. At this point it is very difficult
|
|
|
|
* to print debug info.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef CONFIG_U3_DART
|
|
|
|
extern unsigned long dart_tablebase;
|
|
|
|
#endif /* CONFIG_U3_DART */
|
|
|
|
|
2005-11-10 10:37:51 +08:00
|
|
|
static unsigned long _SDR1;
|
|
|
|
struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
|
|
|
|
|
2007-06-13 12:52:56 +08:00
|
|
|
struct hash_pte *htab_address;
|
2006-02-21 14:22:55 +08:00
|
|
|
unsigned long htab_size_bytes;
|
2005-07-13 16:11:42 +08:00
|
|
|
unsigned long htab_hash_mask;
|
2009-10-30 13:47:19 +08:00
|
|
|
EXPORT_SYMBOL_GPL(htab_hash_mask);
|
2005-11-07 08:06:55 +08:00
|
|
|
int mmu_linear_psize = MMU_PAGE_4K;
|
|
|
|
int mmu_virtual_psize = MMU_PAGE_4K;
|
2006-06-15 08:45:18 +08:00
|
|
|
int mmu_vmalloc_psize = MMU_PAGE_4K;
|
[POWERPC] vmemmap fixes to use smaller pages
This changes vmemmap to use a different region (region 0xf) of the
address space, and to configure the page size of that region
dynamically at boot.
The problem with the current approach of always using 16M pages is that
it's not well suited to machines that have small amounts of memory such
as small partitions on pseries, or PS3's.
In fact, on the PS3, failure to allocate the 16M page backing vmmemmap
tends to prevent hotplugging the HV's "additional" memory, thus limiting
the available memory even more, from my experience down to something
like 80M total, which makes it really not very useable.
The logic used by my match to choose the vmemmap page size is:
- If 16M pages are available and there's 1G or more RAM at boot,
use that size.
- Else if 64K pages are available, use that
- Else use 4K pages
I've tested on a POWER6 (16M pages) and on an iSeries POWER3 (4K pages)
and it seems to work fine.
Note that I intend to change the way we organize the kernel regions &
SLBs so the actual region will change from 0xf back to something else at
one point, as I simplify the SLB miss handler, but that will be for a
later patch.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-04-30 13:41:48 +08:00
|
|
|
#ifdef CONFIG_SPARSEMEM_VMEMMAP
|
|
|
|
int mmu_vmemmap_psize = MMU_PAGE_4K;
|
|
|
|
#endif
|
2006-06-15 08:45:18 +08:00
|
|
|
int mmu_io_psize = MMU_PAGE_4K;
|
2007-10-11 18:37:10 +08:00
|
|
|
int mmu_kernel_ssize = MMU_SEGSIZE_256M;
|
|
|
|
int mmu_highuser_ssize = MMU_SEGSIZE_256M;
|
2007-12-06 14:24:48 +08:00
|
|
|
u16 mmu_slb_size = 64;
|
2009-10-30 13:47:19 +08:00
|
|
|
EXPORT_SYMBOL_GPL(mmu_slb_size);
|
2006-06-15 08:45:18 +08:00
|
|
|
#ifdef CONFIG_PPC_64K_PAGES
|
|
|
|
int mmu_ci_restrictions;
|
|
|
|
#endif
|
2007-04-12 13:30:23 +08:00
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
|
|
|
static u8 *linear_map_hash_slots;
|
|
|
|
static unsigned long linear_map_hash_count;
|
2007-04-18 09:50:09 +08:00
|
|
|
static DEFINE_SPINLOCK(linear_map_hash_lock);
|
2007-04-12 13:30:23 +08:00
|
|
|
#endif /* CONFIG_DEBUG_PAGEALLOC */
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* There are definitions of page sizes arrays to be used when none
|
|
|
|
* is provided by the firmware.
|
|
|
|
*/
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Pre-POWER4 CPUs (4k pages only)
|
|
|
|
*/
|
2008-05-08 12:27:07 +08:00
|
|
|
static struct mmu_psize_def mmu_psize_defaults_old[] = {
|
2005-11-07 08:06:55 +08:00
|
|
|
[MMU_PAGE_4K] = {
|
|
|
|
.shift = 12,
|
|
|
|
.sllp = 0,
|
2013-04-28 17:37:35 +08:00
|
|
|
.penc = {[MMU_PAGE_4K] = 0, [1 ... MMU_PAGE_COUNT - 1] = -1},
|
2005-11-07 08:06:55 +08:00
|
|
|
.avpnm = 0,
|
|
|
|
.tlbiel = 0,
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
|
|
|
/* POWER4, GPUL, POWER5
|
|
|
|
*
|
|
|
|
* Support for 16Mb large pages
|
|
|
|
*/
|
2008-05-08 12:27:07 +08:00
|
|
|
static struct mmu_psize_def mmu_psize_defaults_gp[] = {
|
2005-11-07 08:06:55 +08:00
|
|
|
[MMU_PAGE_4K] = {
|
|
|
|
.shift = 12,
|
|
|
|
.sllp = 0,
|
2013-04-28 17:37:35 +08:00
|
|
|
.penc = {[MMU_PAGE_4K] = 0, [1 ... MMU_PAGE_COUNT - 1] = -1},
|
2005-11-07 08:06:55 +08:00
|
|
|
.avpnm = 0,
|
|
|
|
.tlbiel = 1,
|
|
|
|
},
|
|
|
|
[MMU_PAGE_16M] = {
|
|
|
|
.shift = 24,
|
|
|
|
.sllp = SLB_VSID_L,
|
2013-04-28 17:37:35 +08:00
|
|
|
.penc = {[0 ... MMU_PAGE_16M - 1] = -1, [MMU_PAGE_16M] = 0,
|
|
|
|
[MMU_PAGE_16M + 1 ... MMU_PAGE_COUNT - 1] = -1 },
|
2005-11-07 08:06:55 +08:00
|
|
|
.avpnm = 0x1UL,
|
|
|
|
.tlbiel = 0,
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
2008-08-05 14:19:56 +08:00
|
|
|
static unsigned long htab_convert_pte_flags(unsigned long pteflags)
|
|
|
|
{
|
|
|
|
unsigned long rflags = pteflags & 0x1fa;
|
|
|
|
|
|
|
|
/* _PAGE_EXEC -> NOEXEC */
|
|
|
|
if ((pteflags & _PAGE_EXEC) == 0)
|
|
|
|
rflags |= HPTE_R_N;
|
|
|
|
|
|
|
|
/* PP bits. PAGE_USER is already PP bit 0x2, so we only
|
|
|
|
* need to add in 0x1 if it's a read-only user page
|
|
|
|
*/
|
|
|
|
if ((pteflags & _PAGE_USER) && !((pteflags & _PAGE_RW) &&
|
|
|
|
(pteflags & _PAGE_DIRTY)))
|
|
|
|
rflags |= 1;
|
2013-11-18 17:28:10 +08:00
|
|
|
/*
|
|
|
|
* Always add "C" bit for perf. Memory coherence is always enabled
|
|
|
|
*/
|
|
|
|
return rflags | HPTE_R_C | HPTE_R_M;
|
2008-08-05 14:19:56 +08:00
|
|
|
}
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
|
2008-08-05 14:19:56 +08:00
|
|
|
unsigned long pstart, unsigned long prot,
|
2007-10-11 18:37:10 +08:00
|
|
|
int psize, int ssize)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned long vaddr, paddr;
|
|
|
|
unsigned int step, shift;
|
|
|
|
int ret = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
shift = mmu_psize_defs[psize].shift;
|
|
|
|
step = 1 << shift;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-08-05 14:19:56 +08:00
|
|
|
prot = htab_convert_pte_flags(prot);
|
|
|
|
|
|
|
|
DBG("htab_bolt_mapping(%lx..%lx -> %lx (%lx,%d,%d)\n",
|
|
|
|
vstart, vend, pstart, prot, psize, ssize);
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
for (vaddr = vstart, paddr = pstart; vaddr < vend;
|
|
|
|
vaddr += step, paddr += step) {
|
2007-04-12 13:30:23 +08:00
|
|
|
unsigned long hash, hpteg;
|
2007-10-11 18:37:10 +08:00
|
|
|
unsigned long vsid = get_kernel_vsid(vaddr, ssize);
|
2012-09-10 10:52:50 +08:00
|
|
|
unsigned long vpn = hpt_vpn(vaddr, vsid, ssize);
|
2008-08-30 09:26:27 +08:00
|
|
|
unsigned long tprot = prot;
|
|
|
|
|
2013-03-13 11:34:54 +08:00
|
|
|
/*
|
|
|
|
* If we hit a bad address return error.
|
|
|
|
*/
|
|
|
|
if (!vsid)
|
|
|
|
return -1;
|
2008-08-30 09:26:27 +08:00
|
|
|
/* Make kernel text executable */
|
powerpc: Make the 64-bit kernel as a position-independent executable
This implements CONFIG_RELOCATABLE for 64-bit by making the kernel as
a position-independent executable (PIE) when it is set. This involves
processing the dynamic relocations in the image in the early stages of
booting, even if the kernel is being run at the address it is linked at,
since the linker does not necessarily fill in words in the image for
which there are dynamic relocations. (In fact the linker does fill in
such words for 64-bit executables, though not for 32-bit executables,
so in principle we could avoid calling relocate() entirely when we're
running a 64-bit kernel at the linked address.)
The dynamic relocations are processed by a new function relocate(addr),
where the addr parameter is the virtual address where the image will be
run. In fact we call it twice; once before calling prom_init, and again
when starting the main kernel. This means that reloc_offset() returns
0 in prom_init (since it has been relocated to the address it is running
at), which necessitated a few adjustments.
This also changes __va and __pa to use an equivalent definition that is
simpler. With the relocatable kernel, PAGE_OFFSET and MEMORY_START are
constants (for 64-bit) whereas PHYSICAL_START is a variable (and
KERNELBASE ideally should be too, but isn't yet).
With this, relocatable kernels still copy themselves down to physical
address 0 and run there.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-08-30 09:43:47 +08:00
|
|
|
if (overlaps_kernel_text(vaddr, vaddr + step))
|
2008-08-30 09:26:27 +08:00
|
|
|
tprot &= ~HPTE_R_N;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
hash = hpt_hash(vpn, shift, ssize);
|
2005-04-17 06:20:36 +08:00
|
|
|
hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
|
|
|
|
|
2006-06-23 16:16:39 +08:00
|
|
|
BUG_ON(!ppc_md.hpte_insert);
|
2012-09-10 10:52:50 +08:00
|
|
|
ret = ppc_md.hpte_insert(hpteg, vpn, paddr, tprot,
|
2013-04-28 17:37:35 +08:00
|
|
|
HPTE_V_BOLTED, psize, psize, ssize);
|
2006-06-23 16:16:39 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
if (ret < 0)
|
|
|
|
break;
|
2007-04-12 13:30:23 +08:00
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
|
|
|
if ((paddr >> PAGE_SHIFT) < linear_map_hash_count)
|
|
|
|
linear_map_hash_slots[paddr >> PAGE_SHIFT] = ret | 0x80;
|
|
|
|
#endif /* CONFIG_DEBUG_PAGEALLOC */
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
|
|
|
return ret < 0 ? ret : 0;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2008-03-27 13:08:57 +08:00
|
|
|
#ifdef CONFIG_MEMORY_HOTPLUG
|
2008-03-28 08:37:21 +08:00
|
|
|
static int htab_remove_mapping(unsigned long vstart, unsigned long vend,
|
2008-01-29 06:19:24 +08:00
|
|
|
int psize, int ssize)
|
|
|
|
{
|
|
|
|
unsigned long vaddr;
|
|
|
|
unsigned int step, shift;
|
|
|
|
|
|
|
|
shift = mmu_psize_defs[psize].shift;
|
|
|
|
step = 1 << shift;
|
|
|
|
|
|
|
|
if (!ppc_md.hpte_removebolted) {
|
2008-03-28 08:37:21 +08:00
|
|
|
printk(KERN_WARNING "Platform doesn't implement "
|
|
|
|
"hpte_removebolted\n");
|
|
|
|
return -EINVAL;
|
2008-01-29 06:19:24 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
for (vaddr = vstart; vaddr < vend; vaddr += step)
|
|
|
|
ppc_md.hpte_removebolted(vaddr, psize, ssize);
|
2008-03-28 08:37:21 +08:00
|
|
|
|
|
|
|
return 0;
|
2008-01-29 06:19:24 +08:00
|
|
|
}
|
2008-03-27 13:08:57 +08:00
|
|
|
#endif /* CONFIG_MEMORY_HOTPLUG */
|
2008-01-29 06:19:24 +08:00
|
|
|
|
2007-10-11 18:37:10 +08:00
|
|
|
static int __init htab_dt_scan_seg_sizes(unsigned long node,
|
|
|
|
const char *uname, int depth,
|
|
|
|
void *data)
|
|
|
|
{
|
|
|
|
char *type = of_get_flat_dt_prop(node, "device_type", NULL);
|
2013-09-23 10:04:36 +08:00
|
|
|
__be32 *prop;
|
2007-10-11 18:37:10 +08:00
|
|
|
unsigned long size = 0;
|
|
|
|
|
|
|
|
/* We are scanning "cpu" nodes only */
|
|
|
|
if (type == NULL || strcmp(type, "cpu") != 0)
|
|
|
|
return 0;
|
|
|
|
|
2013-09-23 10:04:36 +08:00
|
|
|
prop = of_get_flat_dt_prop(node, "ibm,processor-segment-sizes", &size);
|
2007-10-11 18:37:10 +08:00
|
|
|
if (prop == NULL)
|
|
|
|
return 0;
|
|
|
|
for (; size >= 4; size -= 4, ++prop) {
|
2013-09-23 10:04:36 +08:00
|
|
|
if (be32_to_cpu(prop[0]) == 40) {
|
2007-10-11 18:37:10 +08:00
|
|
|
DBG("1T segment support detected\n");
|
2011-04-07 03:48:50 +08:00
|
|
|
cur_cpu_spec->mmu_features |= MMU_FTR_1T_SEGMENT;
|
2007-10-12 14:44:55 +08:00
|
|
|
return 1;
|
2007-10-11 18:37:10 +08:00
|
|
|
}
|
|
|
|
}
|
2011-04-07 03:48:50 +08:00
|
|
|
cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
|
2007-10-11 18:37:10 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __init htab_init_seg_sizes(void)
|
|
|
|
{
|
|
|
|
of_scan_flat_dt(htab_dt_scan_seg_sizes, NULL);
|
|
|
|
}
|
|
|
|
|
2013-04-28 17:37:35 +08:00
|
|
|
static int __init get_idx_from_shift(unsigned int shift)
|
|
|
|
{
|
|
|
|
int idx = -1;
|
|
|
|
|
|
|
|
switch (shift) {
|
|
|
|
case 0xc:
|
|
|
|
idx = MMU_PAGE_4K;
|
|
|
|
break;
|
|
|
|
case 0x10:
|
|
|
|
idx = MMU_PAGE_64K;
|
|
|
|
break;
|
|
|
|
case 0x14:
|
|
|
|
idx = MMU_PAGE_1M;
|
|
|
|
break;
|
|
|
|
case 0x18:
|
|
|
|
idx = MMU_PAGE_16M;
|
|
|
|
break;
|
|
|
|
case 0x22:
|
|
|
|
idx = MMU_PAGE_16G;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return idx;
|
|
|
|
}
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
static int __init htab_dt_scan_page_sizes(unsigned long node,
|
|
|
|
const char *uname, int depth,
|
|
|
|
void *data)
|
|
|
|
{
|
|
|
|
char *type = of_get_flat_dt_prop(node, "device_type", NULL);
|
2013-09-23 10:04:36 +08:00
|
|
|
__be32 *prop;
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned long size = 0;
|
|
|
|
|
|
|
|
/* We are scanning "cpu" nodes only */
|
|
|
|
if (type == NULL || strcmp(type, "cpu") != 0)
|
|
|
|
return 0;
|
|
|
|
|
2013-09-23 10:04:36 +08:00
|
|
|
prop = of_get_flat_dt_prop(node, "ibm,segment-page-sizes", &size);
|
2005-11-07 08:06:55 +08:00
|
|
|
if (prop != NULL) {
|
2013-04-28 17:37:38 +08:00
|
|
|
pr_info("Page sizes from device-tree:\n");
|
2005-11-07 08:06:55 +08:00
|
|
|
size /= 4;
|
2011-04-07 03:48:50 +08:00
|
|
|
cur_cpu_spec->mmu_features &= ~(MMU_FTR_16M_PAGE);
|
2005-11-07 08:06:55 +08:00
|
|
|
while(size > 0) {
|
2013-09-23 10:04:36 +08:00
|
|
|
unsigned int base_shift = be32_to_cpu(prop[0]);
|
|
|
|
unsigned int slbenc = be32_to_cpu(prop[1]);
|
|
|
|
unsigned int lpnum = be32_to_cpu(prop[2]);
|
2005-11-07 08:06:55 +08:00
|
|
|
struct mmu_psize_def *def;
|
2013-04-28 17:37:35 +08:00
|
|
|
int idx, base_idx;
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
size -= 3; prop += 3;
|
2013-04-28 17:37:35 +08:00
|
|
|
base_idx = get_idx_from_shift(base_shift);
|
|
|
|
if (base_idx < 0) {
|
|
|
|
/*
|
|
|
|
* skip the pte encoding also
|
|
|
|
*/
|
|
|
|
prop += lpnum * 2; size -= lpnum * 2;
|
|
|
|
continue;
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
2013-04-28 17:37:35 +08:00
|
|
|
def = &mmu_psize_defs[base_idx];
|
|
|
|
if (base_idx == MMU_PAGE_16M)
|
2011-04-07 03:48:50 +08:00
|
|
|
cur_cpu_spec->mmu_features |= MMU_FTR_16M_PAGE;
|
2013-04-28 17:37:35 +08:00
|
|
|
|
|
|
|
def->shift = base_shift;
|
|
|
|
if (base_shift <= 23)
|
2005-11-07 08:06:55 +08:00
|
|
|
def->avpnm = 0;
|
|
|
|
else
|
2013-04-28 17:37:35 +08:00
|
|
|
def->avpnm = (1 << (base_shift - 23)) - 1;
|
2005-11-07 08:06:55 +08:00
|
|
|
def->sllp = slbenc;
|
2013-04-28 17:37:35 +08:00
|
|
|
/*
|
|
|
|
* We don't know for sure what's up with tlbiel, so
|
2005-11-07 08:06:55 +08:00
|
|
|
* for now we only set it for 4K and 64K pages
|
|
|
|
*/
|
2013-04-28 17:37:35 +08:00
|
|
|
if (base_idx == MMU_PAGE_4K || base_idx == MMU_PAGE_64K)
|
2005-11-07 08:06:55 +08:00
|
|
|
def->tlbiel = 1;
|
|
|
|
else
|
|
|
|
def->tlbiel = 0;
|
|
|
|
|
2013-04-28 17:37:35 +08:00
|
|
|
while (size > 0 && lpnum) {
|
2013-09-23 10:04:36 +08:00
|
|
|
unsigned int shift = be32_to_cpu(prop[0]);
|
|
|
|
int penc = be32_to_cpu(prop[1]);
|
2013-04-28 17:37:35 +08:00
|
|
|
|
|
|
|
prop += 2; size -= 2;
|
|
|
|
lpnum--;
|
|
|
|
|
|
|
|
idx = get_idx_from_shift(shift);
|
|
|
|
if (idx < 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (penc == -1)
|
|
|
|
pr_err("Invalid penc for base_shift=%d "
|
|
|
|
"shift=%d\n", base_shift, shift);
|
|
|
|
|
|
|
|
def->penc[idx] = penc;
|
2013-04-28 17:37:38 +08:00
|
|
|
pr_info("base_shift=%d: shift=%d, sllp=0x%04lx,"
|
|
|
|
" avpnm=0x%08lx, tlbiel=%d, penc=%d\n",
|
|
|
|
base_shift, shift, def->sllp,
|
|
|
|
def->avpnm, def->tlbiel, def->penc[idx]);
|
2013-04-28 17:37:35 +08:00
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2005-11-07 08:06:55 +08:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-07-31 11:51:42 +08:00
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
2008-07-24 12:27:54 +08:00
|
|
|
/* Scan for 16G memory blocks that have been set aside for huge pages
|
|
|
|
* and reserve those blocks for 16G huge pages.
|
|
|
|
*/
|
|
|
|
static int __init htab_dt_scan_hugepage_blocks(unsigned long node,
|
|
|
|
const char *uname, int depth,
|
|
|
|
void *data) {
|
|
|
|
char *type = of_get_flat_dt_prop(node, "device_type", NULL);
|
2013-09-23 10:04:36 +08:00
|
|
|
__be64 *addr_prop;
|
|
|
|
__be32 *page_count_prop;
|
2008-07-24 12:27:54 +08:00
|
|
|
unsigned int expected_pages;
|
|
|
|
long unsigned int phys_addr;
|
|
|
|
long unsigned int block_size;
|
|
|
|
|
|
|
|
/* We are scanning "memory" nodes only */
|
|
|
|
if (type == NULL || strcmp(type, "memory") != 0)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* This property is the log base 2 of the number of virtual pages that
|
|
|
|
* will represent this memory block. */
|
|
|
|
page_count_prop = of_get_flat_dt_prop(node, "ibm,expected#pages", NULL);
|
|
|
|
if (page_count_prop == NULL)
|
|
|
|
return 0;
|
2013-09-23 10:04:36 +08:00
|
|
|
expected_pages = (1 << be32_to_cpu(page_count_prop[0]));
|
2008-07-24 12:27:54 +08:00
|
|
|
addr_prop = of_get_flat_dt_prop(node, "reg", NULL);
|
|
|
|
if (addr_prop == NULL)
|
|
|
|
return 0;
|
2013-09-23 10:04:36 +08:00
|
|
|
phys_addr = be64_to_cpu(addr_prop[0]);
|
|
|
|
block_size = be64_to_cpu(addr_prop[1]);
|
2008-07-24 12:27:54 +08:00
|
|
|
if (block_size != (16 * GB))
|
|
|
|
return 0;
|
|
|
|
printk(KERN_INFO "Huge page(16GB) memory: "
|
|
|
|
"addr = 0x%lX size = 0x%lX pages = %d\n",
|
|
|
|
phys_addr, block_size, expected_pages);
|
2010-07-12 12:36:09 +08:00
|
|
|
if (phys_addr + (16 * GB) <= memblock_end_of_DRAM()) {
|
|
|
|
memblock_reserve(phys_addr, block_size * expected_pages);
|
2008-10-21 23:27:36 +08:00
|
|
|
add_gpage(phys_addr, block_size, expected_pages);
|
|
|
|
}
|
2008-07-24 12:27:54 +08:00
|
|
|
return 0;
|
|
|
|
}
|
2008-07-31 11:51:42 +08:00
|
|
|
#endif /* CONFIG_HUGETLB_PAGE */
|
2008-07-24 12:27:54 +08:00
|
|
|
|
2013-04-28 17:37:35 +08:00
|
|
|
static void mmu_psize_set_default_penc(void)
|
|
|
|
{
|
|
|
|
int bpsize, apsize;
|
|
|
|
for (bpsize = 0; bpsize < MMU_PAGE_COUNT; bpsize++)
|
|
|
|
for (apsize = 0; apsize < MMU_PAGE_COUNT; apsize++)
|
|
|
|
mmu_psize_defs[bpsize].penc[apsize] = -1;
|
|
|
|
}
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
static void __init htab_init_page_sizes(void)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
2013-04-28 17:37:35 +08:00
|
|
|
/* se the invalid penc to -1 */
|
|
|
|
mmu_psize_set_default_penc();
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Default to 4K pages only */
|
|
|
|
memcpy(mmu_psize_defs, mmu_psize_defaults_old,
|
|
|
|
sizeof(mmu_psize_defaults_old));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Try to find the available page sizes in the device-tree
|
|
|
|
*/
|
|
|
|
rc = of_scan_flat_dt(htab_dt_scan_page_sizes, NULL);
|
|
|
|
if (rc != 0) /* Found */
|
|
|
|
goto found;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Not in the device-tree, let's fallback on known size
|
|
|
|
* list for 16M capable GP & GR
|
|
|
|
*/
|
2011-04-07 03:48:50 +08:00
|
|
|
if (mmu_has_feature(MMU_FTR_16M_PAGE))
|
2005-11-07 08:06:55 +08:00
|
|
|
memcpy(mmu_psize_defs, mmu_psize_defaults_gp,
|
|
|
|
sizeof(mmu_psize_defaults_gp));
|
|
|
|
found:
|
2007-04-12 13:30:23 +08:00
|
|
|
#ifndef CONFIG_DEBUG_PAGEALLOC
|
2005-11-07 08:06:55 +08:00
|
|
|
/*
|
|
|
|
* Pick a size for the linear mapping. Currently, we only support
|
|
|
|
* 16M, 1M and 4K which is the default
|
|
|
|
*/
|
|
|
|
if (mmu_psize_defs[MMU_PAGE_16M].shift)
|
|
|
|
mmu_linear_psize = MMU_PAGE_16M;
|
|
|
|
else if (mmu_psize_defs[MMU_PAGE_1M].shift)
|
|
|
|
mmu_linear_psize = MMU_PAGE_1M;
|
2007-04-12 13:30:23 +08:00
|
|
|
#endif /* CONFIG_DEBUG_PAGEALLOC */
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2006-06-15 08:45:18 +08:00
|
|
|
#ifdef CONFIG_PPC_64K_PAGES
|
2005-11-07 08:06:55 +08:00
|
|
|
/*
|
|
|
|
* Pick a size for the ordinary pages. Default is 4K, we support
|
2006-06-15 08:45:18 +08:00
|
|
|
* 64K for user mappings and vmalloc if supported by the processor.
|
|
|
|
* We only use 64k for ioremap if the processor
|
|
|
|
* (and firmware) support cache-inhibited large pages.
|
|
|
|
* If not, we use 4k and set mmu_ci_restrictions so that
|
|
|
|
* hash_page knows to switch processes that use cache-inhibited
|
|
|
|
* mappings to 4k pages.
|
2005-11-07 08:06:55 +08:00
|
|
|
*/
|
2006-06-15 08:45:18 +08:00
|
|
|
if (mmu_psize_defs[MMU_PAGE_64K].shift) {
|
2005-11-07 08:06:55 +08:00
|
|
|
mmu_virtual_psize = MMU_PAGE_64K;
|
2006-06-15 08:45:18 +08:00
|
|
|
mmu_vmalloc_psize = MMU_PAGE_64K;
|
2007-04-12 13:30:23 +08:00
|
|
|
if (mmu_linear_psize == MMU_PAGE_4K)
|
|
|
|
mmu_linear_psize = MMU_PAGE_64K;
|
2011-04-07 03:48:50 +08:00
|
|
|
if (mmu_has_feature(MMU_FTR_CI_LARGE_PAGE)) {
|
2008-03-24 14:41:22 +08:00
|
|
|
/*
|
|
|
|
* Don't use 64k pages for ioremap on pSeries, since
|
|
|
|
* that would stop us accessing the HEA ethernet.
|
|
|
|
*/
|
|
|
|
if (!machine_is(pseries))
|
|
|
|
mmu_io_psize = MMU_PAGE_64K;
|
|
|
|
} else
|
2006-06-15 08:45:18 +08:00
|
|
|
mmu_ci_restrictions = 1;
|
|
|
|
}
|
2007-04-12 13:30:23 +08:00
|
|
|
#endif /* CONFIG_PPC_64K_PAGES */
|
2005-11-07 08:06:55 +08:00
|
|
|
|
[POWERPC] vmemmap fixes to use smaller pages
This changes vmemmap to use a different region (region 0xf) of the
address space, and to configure the page size of that region
dynamically at boot.
The problem with the current approach of always using 16M pages is that
it's not well suited to machines that have small amounts of memory such
as small partitions on pseries, or PS3's.
In fact, on the PS3, failure to allocate the 16M page backing vmmemmap
tends to prevent hotplugging the HV's "additional" memory, thus limiting
the available memory even more, from my experience down to something
like 80M total, which makes it really not very useable.
The logic used by my match to choose the vmemmap page size is:
- If 16M pages are available and there's 1G or more RAM at boot,
use that size.
- Else if 64K pages are available, use that
- Else use 4K pages
I've tested on a POWER6 (16M pages) and on an iSeries POWER3 (4K pages)
and it seems to work fine.
Note that I intend to change the way we organize the kernel regions &
SLBs so the actual region will change from 0xf back to something else at
one point, as I simplify the SLB miss handler, but that will be for a
later patch.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-04-30 13:41:48 +08:00
|
|
|
#ifdef CONFIG_SPARSEMEM_VMEMMAP
|
|
|
|
/* We try to use 16M pages for vmemmap if that is supported
|
|
|
|
* and we have at least 1G of RAM at boot
|
|
|
|
*/
|
|
|
|
if (mmu_psize_defs[MMU_PAGE_16M].shift &&
|
2010-07-12 12:36:09 +08:00
|
|
|
memblock_phys_mem_size() >= 0x40000000)
|
[POWERPC] vmemmap fixes to use smaller pages
This changes vmemmap to use a different region (region 0xf) of the
address space, and to configure the page size of that region
dynamically at boot.
The problem with the current approach of always using 16M pages is that
it's not well suited to machines that have small amounts of memory such
as small partitions on pseries, or PS3's.
In fact, on the PS3, failure to allocate the 16M page backing vmmemmap
tends to prevent hotplugging the HV's "additional" memory, thus limiting
the available memory even more, from my experience down to something
like 80M total, which makes it really not very useable.
The logic used by my match to choose the vmemmap page size is:
- If 16M pages are available and there's 1G or more RAM at boot,
use that size.
- Else if 64K pages are available, use that
- Else use 4K pages
I've tested on a POWER6 (16M pages) and on an iSeries POWER3 (4K pages)
and it seems to work fine.
Note that I intend to change the way we organize the kernel regions &
SLBs so the actual region will change from 0xf back to something else at
one point, as I simplify the SLB miss handler, but that will be for a
later patch.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-04-30 13:41:48 +08:00
|
|
|
mmu_vmemmap_psize = MMU_PAGE_16M;
|
|
|
|
else if (mmu_psize_defs[MMU_PAGE_64K].shift)
|
|
|
|
mmu_vmemmap_psize = MMU_PAGE_64K;
|
|
|
|
else
|
|
|
|
mmu_vmemmap_psize = MMU_PAGE_4K;
|
|
|
|
#endif /* CONFIG_SPARSEMEM_VMEMMAP */
|
|
|
|
|
2006-06-15 08:45:18 +08:00
|
|
|
printk(KERN_DEBUG "Page orders: linear mapping = %d, "
|
[POWERPC] vmemmap fixes to use smaller pages
This changes vmemmap to use a different region (region 0xf) of the
address space, and to configure the page size of that region
dynamically at boot.
The problem with the current approach of always using 16M pages is that
it's not well suited to machines that have small amounts of memory such
as small partitions on pseries, or PS3's.
In fact, on the PS3, failure to allocate the 16M page backing vmmemmap
tends to prevent hotplugging the HV's "additional" memory, thus limiting
the available memory even more, from my experience down to something
like 80M total, which makes it really not very useable.
The logic used by my match to choose the vmemmap page size is:
- If 16M pages are available and there's 1G or more RAM at boot,
use that size.
- Else if 64K pages are available, use that
- Else use 4K pages
I've tested on a POWER6 (16M pages) and on an iSeries POWER3 (4K pages)
and it seems to work fine.
Note that I intend to change the way we organize the kernel regions &
SLBs so the actual region will change from 0xf back to something else at
one point, as I simplify the SLB miss handler, but that will be for a
later patch.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-04-30 13:41:48 +08:00
|
|
|
"virtual = %d, io = %d"
|
|
|
|
#ifdef CONFIG_SPARSEMEM_VMEMMAP
|
|
|
|
", vmemmap = %d"
|
|
|
|
#endif
|
|
|
|
"\n",
|
2005-11-07 08:06:55 +08:00
|
|
|
mmu_psize_defs[mmu_linear_psize].shift,
|
2006-06-15 08:45:18 +08:00
|
|
|
mmu_psize_defs[mmu_virtual_psize].shift,
|
[POWERPC] vmemmap fixes to use smaller pages
This changes vmemmap to use a different region (region 0xf) of the
address space, and to configure the page size of that region
dynamically at boot.
The problem with the current approach of always using 16M pages is that
it's not well suited to machines that have small amounts of memory such
as small partitions on pseries, or PS3's.
In fact, on the PS3, failure to allocate the 16M page backing vmmemmap
tends to prevent hotplugging the HV's "additional" memory, thus limiting
the available memory even more, from my experience down to something
like 80M total, which makes it really not very useable.
The logic used by my match to choose the vmemmap page size is:
- If 16M pages are available and there's 1G or more RAM at boot,
use that size.
- Else if 64K pages are available, use that
- Else use 4K pages
I've tested on a POWER6 (16M pages) and on an iSeries POWER3 (4K pages)
and it seems to work fine.
Note that I intend to change the way we organize the kernel regions &
SLBs so the actual region will change from 0xf back to something else at
one point, as I simplify the SLB miss handler, but that will be for a
later patch.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-04-30 13:41:48 +08:00
|
|
|
mmu_psize_defs[mmu_io_psize].shift
|
|
|
|
#ifdef CONFIG_SPARSEMEM_VMEMMAP
|
|
|
|
,mmu_psize_defs[mmu_vmemmap_psize].shift
|
|
|
|
#endif
|
|
|
|
);
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
2008-07-24 12:27:54 +08:00
|
|
|
/* Reserve 16G huge page memory sections for huge pages */
|
|
|
|
of_scan_flat_dt(htab_dt_scan_hugepage_blocks, NULL);
|
2005-11-07 08:06:55 +08:00
|
|
|
#endif /* CONFIG_HUGETLB_PAGE */
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init htab_dt_scan_pftsize(unsigned long node,
|
|
|
|
const char *uname, int depth,
|
|
|
|
void *data)
|
|
|
|
{
|
|
|
|
char *type = of_get_flat_dt_prop(node, "device_type", NULL);
|
2013-09-23 10:04:36 +08:00
|
|
|
__be32 *prop;
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
/* We are scanning "cpu" nodes only */
|
|
|
|
if (type == NULL || strcmp(type, "cpu") != 0)
|
|
|
|
return 0;
|
|
|
|
|
2013-09-23 10:04:36 +08:00
|
|
|
prop = of_get_flat_dt_prop(node, "ibm,pft-size", NULL);
|
2005-11-07 08:06:55 +08:00
|
|
|
if (prop != NULL) {
|
|
|
|
/* pft_size[0] is the NUMA CEC cookie */
|
2013-09-23 10:04:36 +08:00
|
|
|
ppc64_pft_size = be32_to_cpu(prop[1]);
|
2005-11-07 08:06:55 +08:00
|
|
|
return 1;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2005-11-07 08:06:55 +08:00
|
|
|
return 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
static unsigned long __init htab_get_table_size(void)
|
2005-10-12 14:58:53 +08:00
|
|
|
{
|
2009-02-13 19:57:30 +08:00
|
|
|
unsigned long mem_size, rnd_mem_size, pteg_count, psize;
|
2005-10-12 14:58:53 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* If hash size isn't already provided by the platform, we try to
|
2006-01-10 07:10:13 +08:00
|
|
|
* retrieve it from the device-tree. If it's not there neither, we
|
2005-11-07 08:06:55 +08:00
|
|
|
* calculate it now based on the total RAM size
|
2005-10-12 14:58:53 +08:00
|
|
|
*/
|
2005-11-07 08:06:55 +08:00
|
|
|
if (ppc64_pft_size == 0)
|
|
|
|
of_scan_flat_dt(htab_dt_scan_pftsize, NULL);
|
2005-10-12 14:58:53 +08:00
|
|
|
if (ppc64_pft_size)
|
|
|
|
return 1UL << ppc64_pft_size;
|
|
|
|
|
|
|
|
/* round mem_size up to next power of 2 */
|
2010-07-12 12:36:09 +08:00
|
|
|
mem_size = memblock_phys_mem_size();
|
2005-11-10 10:37:51 +08:00
|
|
|
rnd_mem_size = 1UL << __ilog2(mem_size);
|
|
|
|
if (rnd_mem_size < mem_size)
|
2005-10-12 14:58:53 +08:00
|
|
|
rnd_mem_size <<= 1;
|
|
|
|
|
|
|
|
/* # pages / 2 */
|
2009-02-13 19:57:30 +08:00
|
|
|
psize = mmu_psize_defs[mmu_virtual_psize].shift;
|
|
|
|
pteg_count = max(rnd_mem_size >> (psize + 1), 1UL << 11);
|
2005-10-12 14:58:53 +08:00
|
|
|
|
|
|
|
return pteg_count << 7;
|
|
|
|
}
|
|
|
|
|
2005-11-08 08:25:48 +08:00
|
|
|
#ifdef CONFIG_MEMORY_HOTPLUG
|
2011-08-11 04:44:24 +08:00
|
|
|
int create_section_mapping(unsigned long start, unsigned long end)
|
2005-11-08 08:25:48 +08:00
|
|
|
{
|
2011-08-11 04:44:24 +08:00
|
|
|
return htab_bolt_mapping(start, end, __pa(start),
|
2008-10-13 01:54:24 +08:00
|
|
|
pgprot_val(PAGE_KERNEL), mmu_linear_psize,
|
2011-08-11 04:44:24 +08:00
|
|
|
mmu_kernel_ssize);
|
2005-11-08 08:25:48 +08:00
|
|
|
}
|
2008-01-29 06:19:24 +08:00
|
|
|
|
2008-03-28 08:37:21 +08:00
|
|
|
int remove_section_mapping(unsigned long start, unsigned long end)
|
2008-01-29 06:19:24 +08:00
|
|
|
{
|
2008-03-28 08:37:21 +08:00
|
|
|
return htab_remove_mapping(start, end, mmu_linear_psize,
|
|
|
|
mmu_kernel_ssize);
|
2008-01-29 06:19:24 +08:00
|
|
|
}
|
2005-11-08 08:25:48 +08:00
|
|
|
#endif /* CONFIG_MEMORY_HOTPLUG */
|
|
|
|
|
2011-04-05 07:56:18 +08:00
|
|
|
#define FUNCTION_TEXT(A) ((*(unsigned long *)(A)))
|
2006-06-23 16:16:38 +08:00
|
|
|
|
|
|
|
static void __init htab_finish_init(void)
|
|
|
|
{
|
|
|
|
extern unsigned int *htab_call_hpte_insert1;
|
|
|
|
extern unsigned int *htab_call_hpte_insert2;
|
|
|
|
extern unsigned int *htab_call_hpte_remove;
|
|
|
|
extern unsigned int *htab_call_hpte_updatepp;
|
|
|
|
|
2007-05-08 14:27:28 +08:00
|
|
|
#ifdef CONFIG_PPC_HAS_HASH_64K
|
2006-06-23 16:16:38 +08:00
|
|
|
extern unsigned int *ht64_call_hpte_insert1;
|
|
|
|
extern unsigned int *ht64_call_hpte_insert2;
|
|
|
|
extern unsigned int *ht64_call_hpte_remove;
|
|
|
|
extern unsigned int *ht64_call_hpte_updatepp;
|
|
|
|
|
2011-04-05 07:56:18 +08:00
|
|
|
patch_branch(ht64_call_hpte_insert1,
|
|
|
|
FUNCTION_TEXT(ppc_md.hpte_insert),
|
|
|
|
BRANCH_SET_LINK);
|
|
|
|
patch_branch(ht64_call_hpte_insert2,
|
|
|
|
FUNCTION_TEXT(ppc_md.hpte_insert),
|
|
|
|
BRANCH_SET_LINK);
|
|
|
|
patch_branch(ht64_call_hpte_remove,
|
|
|
|
FUNCTION_TEXT(ppc_md.hpte_remove),
|
|
|
|
BRANCH_SET_LINK);
|
|
|
|
patch_branch(ht64_call_hpte_updatepp,
|
|
|
|
FUNCTION_TEXT(ppc_md.hpte_updatepp),
|
|
|
|
BRANCH_SET_LINK);
|
|
|
|
|
2007-05-17 02:43:02 +08:00
|
|
|
#endif /* CONFIG_PPC_HAS_HASH_64K */
|
2006-06-23 16:16:38 +08:00
|
|
|
|
2011-04-05 07:56:18 +08:00
|
|
|
patch_branch(htab_call_hpte_insert1,
|
|
|
|
FUNCTION_TEXT(ppc_md.hpte_insert),
|
|
|
|
BRANCH_SET_LINK);
|
|
|
|
patch_branch(htab_call_hpte_insert2,
|
|
|
|
FUNCTION_TEXT(ppc_md.hpte_insert),
|
|
|
|
BRANCH_SET_LINK);
|
|
|
|
patch_branch(htab_call_hpte_remove,
|
|
|
|
FUNCTION_TEXT(ppc_md.hpte_remove),
|
|
|
|
BRANCH_SET_LINK);
|
|
|
|
patch_branch(htab_call_hpte_updatepp,
|
|
|
|
FUNCTION_TEXT(ppc_md.hpte_updatepp),
|
|
|
|
BRANCH_SET_LINK);
|
2006-06-23 16:16:38 +08:00
|
|
|
}
|
|
|
|
|
2009-03-20 03:34:16 +08:00
|
|
|
static void __init htab_initialize(void)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2006-02-21 14:22:55 +08:00
|
|
|
unsigned long table;
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned long pteg_count;
|
2008-08-30 09:26:27 +08:00
|
|
|
unsigned long prot;
|
2008-01-29 22:13:59 +08:00
|
|
|
unsigned long base = 0, size = 0, limit;
|
2010-08-04 11:43:53 +08:00
|
|
|
struct memblock_region *reg;
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
DBG(" -> htab_initialize()\n");
|
|
|
|
|
2007-10-11 18:37:10 +08:00
|
|
|
/* Initialize segment sizes */
|
|
|
|
htab_init_seg_sizes();
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Initialize page sizes */
|
|
|
|
htab_init_page_sizes();
|
|
|
|
|
2011-04-07 03:48:50 +08:00
|
|
|
if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) {
|
2007-10-11 18:37:10 +08:00
|
|
|
mmu_kernel_ssize = MMU_SEGSIZE_1T;
|
|
|
|
mmu_highuser_ssize = MMU_SEGSIZE_1T;
|
|
|
|
printk(KERN_INFO "Using 1TB segments\n");
|
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Calculate the required size of the htab. We want the number of
|
|
|
|
* PTEGs to equal one half the number of real pages.
|
|
|
|
*/
|
2005-11-07 08:06:55 +08:00
|
|
|
htab_size_bytes = htab_get_table_size();
|
2005-04-17 06:20:36 +08:00
|
|
|
pteg_count = htab_size_bytes >> 7;
|
|
|
|
|
|
|
|
htab_hash_mask = pteg_count - 1;
|
|
|
|
|
2006-03-21 17:45:59 +08:00
|
|
|
if (firmware_has_feature(FW_FEATURE_LPAR)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Using a hypervisor which owns the htab */
|
|
|
|
htab_address = NULL;
|
|
|
|
_SDR1 = 0;
|
2012-02-20 10:15:03 +08:00
|
|
|
#ifdef CONFIG_FA_DUMP
|
|
|
|
/*
|
|
|
|
* If firmware assisted dump is active firmware preserves
|
|
|
|
* the contents of htab along with entire partition memory.
|
|
|
|
* Clear the htab if firmware assisted dump is active so
|
|
|
|
* that we dont end up using old mappings.
|
|
|
|
*/
|
|
|
|
if (is_fadump_active() && ppc_md.hpte_clear_all)
|
|
|
|
ppc_md.hpte_clear_all();
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
} else {
|
|
|
|
/* Find storage for the HPT. Must be contiguous in
|
2008-01-29 22:13:59 +08:00
|
|
|
* the absolute address space. On cell we want it to be
|
2008-03-12 15:03:24 +08:00
|
|
|
* in the first 2 Gig so we can use it for IOMMU hacks.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
2008-01-29 22:13:59 +08:00
|
|
|
if (machine_is(cell))
|
2008-03-12 15:03:24 +08:00
|
|
|
limit = 0x80000000;
|
2008-01-29 22:13:59 +08:00
|
|
|
else
|
2010-07-07 06:39:00 +08:00
|
|
|
limit = MEMBLOCK_ALLOC_ANYWHERE;
|
2008-01-29 22:13:59 +08:00
|
|
|
|
2010-07-12 12:36:09 +08:00
|
|
|
table = memblock_alloc_base(htab_size_bytes, htab_size_bytes, limit);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
DBG("Hash table allocated at %lx, size: %lx\n", table,
|
|
|
|
htab_size_bytes);
|
|
|
|
|
2012-07-26 05:19:50 +08:00
|
|
|
htab_address = __va(table);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/* htab absolute addr + encoded htabsize */
|
|
|
|
_SDR1 = table + __ilog2(pteg_count) - 11;
|
|
|
|
|
|
|
|
/* Initialize the HPT with no entries */
|
|
|
|
memset((void *)table, 0, htab_size_bytes);
|
2005-11-10 10:37:51 +08:00
|
|
|
|
|
|
|
/* Set SDR1 */
|
|
|
|
mtspr(SPRN_SDR1, _SDR1);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2008-10-13 01:54:24 +08:00
|
|
|
prot = pgprot_val(PAGE_KERNEL);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-04-12 13:30:23 +08:00
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
2010-07-12 12:36:09 +08:00
|
|
|
linear_map_hash_count = memblock_end_of_DRAM() >> PAGE_SHIFT;
|
|
|
|
linear_map_hash_slots = __va(memblock_alloc_base(linear_map_hash_count,
|
2010-07-07 06:39:02 +08:00
|
|
|
1, ppc64_rma_size));
|
2007-04-12 13:30:23 +08:00
|
|
|
memset(linear_map_hash_slots, 0, linear_map_hash_count);
|
|
|
|
#endif /* CONFIG_DEBUG_PAGEALLOC */
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* On U3 based machines, we need to reserve the DART area and
|
|
|
|
* _NOT_ map it to avoid cache paradoxes as it's remapped non
|
|
|
|
* cacheable later on
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* create bolted the linear mapping in the hash table */
|
2010-08-04 11:43:53 +08:00
|
|
|
for_each_memblock(memory, reg) {
|
|
|
|
base = (unsigned long)__va(reg->base);
|
|
|
|
size = reg->size;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2009-12-14 05:15:12 +08:00
|
|
|
DBG("creating mapping for region: %lx..%lx (prot: %lx)\n",
|
2008-08-30 09:26:27 +08:00
|
|
|
base, size, prot);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_U3_DART
|
|
|
|
/* Do not map the DART space. Fortunately, it will be aligned
|
2010-07-12 12:36:09 +08:00
|
|
|
* in such a way that it will not cross two memblock regions and
|
2005-11-07 08:06:55 +08:00
|
|
|
* will fit within a single 16Mb page.
|
|
|
|
* The DART space is assumed to be a full 16Mb region even if
|
|
|
|
* we only use 2Mb of that space. We will use more of it later
|
|
|
|
* for AGP GART. We have to use a full 16Mb large page.
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
DBG("DART base: %lx\n", dart_tablebase);
|
|
|
|
|
|
|
|
if (dart_tablebase != 0 && dart_tablebase >= base
|
|
|
|
&& dart_tablebase < (base + size)) {
|
2006-03-21 17:45:51 +08:00
|
|
|
unsigned long dart_table_end = dart_tablebase + 16 * MB;
|
2005-04-17 06:20:36 +08:00
|
|
|
if (base != dart_tablebase)
|
2005-11-07 08:06:55 +08:00
|
|
|
BUG_ON(htab_bolt_mapping(base, dart_tablebase,
|
2008-08-30 09:26:27 +08:00
|
|
|
__pa(base), prot,
|
2007-10-11 18:37:10 +08:00
|
|
|
mmu_linear_psize,
|
|
|
|
mmu_kernel_ssize));
|
2006-03-21 17:45:51 +08:00
|
|
|
if ((base + size) > dart_table_end)
|
2005-11-07 08:06:55 +08:00
|
|
|
BUG_ON(htab_bolt_mapping(dart_tablebase+16*MB,
|
2006-03-21 17:45:51 +08:00
|
|
|
base + size,
|
|
|
|
__pa(dart_table_end),
|
2008-08-30 09:26:27 +08:00
|
|
|
prot,
|
2007-10-11 18:37:10 +08:00
|
|
|
mmu_linear_psize,
|
|
|
|
mmu_kernel_ssize));
|
2005-04-17 06:20:36 +08:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_U3_DART */
|
2006-03-21 17:45:51 +08:00
|
|
|
BUG_ON(htab_bolt_mapping(base, base + size, __pa(base),
|
2008-08-30 09:26:27 +08:00
|
|
|
prot, mmu_linear_psize, mmu_kernel_ssize));
|
2010-07-07 06:39:01 +08:00
|
|
|
}
|
|
|
|
memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If we have a memory_limit and we've allocated TCEs then we need to
|
|
|
|
* explicitly map the TCE area at the top of RAM. We also cope with the
|
|
|
|
* case that the TCEs start below memory_limit.
|
|
|
|
* tce_alloc_start/end are 16MB aligned so the mapping should work
|
|
|
|
* for either 4K or 16MB pages.
|
|
|
|
*/
|
|
|
|
if (tce_alloc_start) {
|
2005-12-06 00:24:33 +08:00
|
|
|
tce_alloc_start = (unsigned long)__va(tce_alloc_start);
|
|
|
|
tce_alloc_end = (unsigned long)__va(tce_alloc_end);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
if (base + size >= tce_alloc_start)
|
|
|
|
tce_alloc_start = base + size + 1;
|
|
|
|
|
2006-03-21 17:45:51 +08:00
|
|
|
BUG_ON(htab_bolt_mapping(tce_alloc_start, tce_alloc_end,
|
2008-08-05 14:19:56 +08:00
|
|
|
__pa(tce_alloc_start), prot,
|
2007-10-11 18:37:10 +08:00
|
|
|
mmu_linear_psize, mmu_kernel_ssize));
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2006-06-23 16:16:38 +08:00
|
|
|
htab_finish_init();
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
DBG(" <- htab_initialize()\n");
|
|
|
|
}
|
|
|
|
#undef KB
|
|
|
|
#undef MB
|
|
|
|
|
2009-03-20 03:34:16 +08:00
|
|
|
void __init early_init_mmu(void)
|
2005-11-10 10:37:51 +08:00
|
|
|
{
|
2009-03-20 03:34:16 +08:00
|
|
|
/* Setup initial STAB address in the PACA */
|
|
|
|
get_paca()->stab_real = __pa((u64)&initial_stab);
|
|
|
|
get_paca()->stab_addr = (u64)&initial_stab;
|
|
|
|
|
|
|
|
/* Initialize the MMU Hash table and create the linear mapping
|
|
|
|
* of memory. Has to be done before stab/slb initialization as
|
|
|
|
* this is currently where the page size encoding is obtained
|
|
|
|
*/
|
|
|
|
htab_initialize();
|
|
|
|
|
2012-03-16 02:18:00 +08:00
|
|
|
/* Initialize stab / SLB management */
|
2011-04-07 03:48:50 +08:00
|
|
|
if (mmu_has_feature(MMU_FTR_SLB))
|
2009-03-20 03:34:16 +08:00
|
|
|
slb_initialize();
|
2013-03-13 06:49:06 +08:00
|
|
|
else
|
|
|
|
stab_initialize(get_paca()->stab_real);
|
2009-03-20 03:34:16 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_SMP
|
2013-06-25 03:30:09 +08:00
|
|
|
void early_init_mmu_secondary(void)
|
2009-03-20 03:34:16 +08:00
|
|
|
{
|
|
|
|
/* Initialize hash table for that CPU */
|
2006-03-21 17:45:59 +08:00
|
|
|
if (!firmware_has_feature(FW_FEATURE_LPAR))
|
2005-11-10 10:37:51 +08:00
|
|
|
mtspr(SPRN_SDR1, _SDR1);
|
2009-03-20 03:34:16 +08:00
|
|
|
|
|
|
|
/* Initialize STAB/SLB. We use a virtual address as it works
|
2012-03-16 02:18:00 +08:00
|
|
|
* in real mode on pSeries.
|
2009-03-20 03:34:16 +08:00
|
|
|
*/
|
2011-04-07 03:48:50 +08:00
|
|
|
if (mmu_has_feature(MMU_FTR_SLB))
|
2009-03-20 03:34:16 +08:00
|
|
|
slb_initialize();
|
|
|
|
else
|
|
|
|
stab_initialize(get_paca()->stab_addr);
|
2005-11-10 10:37:51 +08:00
|
|
|
}
|
2009-03-20 03:34:16 +08:00
|
|
|
#endif /* CONFIG_SMP */
|
2005-11-10 10:37:51 +08:00
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
* Called by asm hashtable.S for doing lazy icache flush
|
|
|
|
*/
|
|
|
|
unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
|
2005-11-08 08:21:05 +08:00
|
|
|
if (!pfn_valid(pte_pfn(pte)))
|
|
|
|
return pp;
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
page = pte_page(pte);
|
|
|
|
|
|
|
|
/* page is dirty */
|
|
|
|
if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
|
|
|
|
if (trap == 0x400) {
|
2009-10-27 03:24:31 +08:00
|
|
|
flush_dcache_icache_page(page);
|
2005-04-17 06:20:36 +08:00
|
|
|
set_bit(PG_arch_1, &page->flags);
|
|
|
|
} else
|
2005-11-07 08:06:55 +08:00
|
|
|
pp |= HPTE_R_N;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
return pp;
|
|
|
|
}
|
|
|
|
|
2008-06-18 13:29:12 +08:00
|
|
|
#ifdef CONFIG_PPC_MM_SLICES
|
|
|
|
unsigned int get_paca_psize(unsigned long addr)
|
|
|
|
{
|
2012-09-10 10:52:52 +08:00
|
|
|
u64 lpsizes;
|
|
|
|
unsigned char *hpsizes;
|
|
|
|
unsigned long index, mask_index;
|
2008-06-18 13:29:12 +08:00
|
|
|
|
|
|
|
if (addr < SLICE_LOW_TOP) {
|
2012-09-10 10:52:52 +08:00
|
|
|
lpsizes = get_paca()->context.low_slices_psize;
|
2008-06-18 13:29:12 +08:00
|
|
|
index = GET_LOW_SLICE_INDEX(addr);
|
2012-09-10 10:52:52 +08:00
|
|
|
return (lpsizes >> (index * 4)) & 0xF;
|
2008-06-18 13:29:12 +08:00
|
|
|
}
|
2012-09-10 10:52:52 +08:00
|
|
|
hpsizes = get_paca()->context.high_slices_psize;
|
|
|
|
index = GET_HIGH_SLICE_INDEX(addr);
|
|
|
|
mask_index = index & 0x1;
|
|
|
|
return (hpsizes[index >> 1] >> (mask_index * 4)) & 0xF;
|
2008-06-18 13:29:12 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
unsigned int get_paca_psize(unsigned long addr)
|
|
|
|
{
|
|
|
|
return get_paca()->context.user_psize;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
[POWERPC] Allow drivers to map individual 4k pages to userspace
Some drivers have resources that they want to be able to map into
userspace that are 4k in size. On a kernel configured with 64k pages
we currently end up mapping the 4k we want plus another 60k of
physical address space, which could contain anything. This can
introduce security problems, for example in the case of an infiniband
adaptor where the other 60k could contain registers that some other
program is using for its communications.
This patch adds a new function, remap_4k_pfn, which drivers can use to
map a single 4k page to userspace regardless of whether the kernel is
using a 4k or a 64k page size. Like remap_pfn_range, it would
typically be called in a driver's mmap function. It only maps a
single 4k page, which on a 64k page kernel appears replicated 16 times
throughout a 64k page. On a 4k page kernel it reduces to a call to
remap_pfn_range.
The way this works on a 64k kernel is that a new bit, _PAGE_4K_PFN,
gets set on the linux PTE. This alters the way that __hash_page_4K
computes the real address to put in the HPTE. The RPN field of the
linux PTE becomes the 4k RPN directly rather than being interpreted as
a 64k RPN. Since the RPN field is 32 bits, this means that physical
addresses being mapped with remap_4k_pfn have to be below 2^44,
i.e. 0x100000000000.
The patch also factors out the code in arch/powerpc/mm/hash_utils_64.c
that deals with demoting a process to use 4k pages into one function
that gets called in the various different places where we need to do
that. There were some discrepancies between exactly what was done in
the various places, such as a call to spu_flush_all_slbs in one case
but not in others.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2007-04-03 19:24:02 +08:00
|
|
|
/*
|
|
|
|
* Demote a segment to using 4k pages.
|
|
|
|
* For now this makes the whole process use 4k pages.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_PPC_64K_PAGES
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
|
2007-05-08 14:27:27 +08:00
|
|
|
{
|
2008-06-18 13:29:12 +08:00
|
|
|
if (get_slice_psize(mm, addr) == MMU_PAGE_4K)
|
[POWERPC] Allow drivers to map individual 4k pages to userspace
Some drivers have resources that they want to be able to map into
userspace that are 4k in size. On a kernel configured with 64k pages
we currently end up mapping the 4k we want plus another 60k of
physical address space, which could contain anything. This can
introduce security problems, for example in the case of an infiniband
adaptor where the other 60k could contain registers that some other
program is using for its communications.
This patch adds a new function, remap_4k_pfn, which drivers can use to
map a single 4k page to userspace regardless of whether the kernel is
using a 4k or a 64k page size. Like remap_pfn_range, it would
typically be called in a driver's mmap function. It only maps a
single 4k page, which on a 64k page kernel appears replicated 16 times
throughout a 64k page. On a 4k page kernel it reduces to a call to
remap_pfn_range.
The way this works on a 64k kernel is that a new bit, _PAGE_4K_PFN,
gets set on the linux PTE. This alters the way that __hash_page_4K
computes the real address to put in the HPTE. The RPN field of the
linux PTE becomes the 4k RPN directly rather than being interpreted as
a 64k RPN. Since the RPN field is 32 bits, this means that physical
addresses being mapped with remap_4k_pfn have to be below 2^44,
i.e. 0x100000000000.
The patch also factors out the code in arch/powerpc/mm/hash_utils_64.c
that deals with demoting a process to use 4k pages into one function
that gets called in the various different places where we need to do
that. There were some discrepancies between exactly what was done in
the various places, such as a call to spu_flush_all_slbs in one case
but not in others.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2007-04-03 19:24:02 +08:00
|
|
|
return;
|
2008-06-18 13:29:12 +08:00
|
|
|
slice_set_range_psize(mm, addr, 1, MMU_PAGE_4K);
|
2007-07-17 00:35:38 +08:00
|
|
|
#ifdef CONFIG_SPU_BASE
|
[POWERPC] Allow drivers to map individual 4k pages to userspace
Some drivers have resources that they want to be able to map into
userspace that are 4k in size. On a kernel configured with 64k pages
we currently end up mapping the 4k we want plus another 60k of
physical address space, which could contain anything. This can
introduce security problems, for example in the case of an infiniband
adaptor where the other 60k could contain registers that some other
program is using for its communications.
This patch adds a new function, remap_4k_pfn, which drivers can use to
map a single 4k page to userspace regardless of whether the kernel is
using a 4k or a 64k page size. Like remap_pfn_range, it would
typically be called in a driver's mmap function. It only maps a
single 4k page, which on a 64k page kernel appears replicated 16 times
throughout a 64k page. On a 4k page kernel it reduces to a call to
remap_pfn_range.
The way this works on a 64k kernel is that a new bit, _PAGE_4K_PFN,
gets set on the linux PTE. This alters the way that __hash_page_4K
computes the real address to put in the HPTE. The RPN field of the
linux PTE becomes the 4k RPN directly rather than being interpreted as
a 64k RPN. Since the RPN field is 32 bits, this means that physical
addresses being mapped with remap_4k_pfn have to be below 2^44,
i.e. 0x100000000000.
The patch also factors out the code in arch/powerpc/mm/hash_utils_64.c
that deals with demoting a process to use 4k pages into one function
that gets called in the various different places where we need to do
that. There were some discrepancies between exactly what was done in
the various places, such as a call to spu_flush_all_slbs in one case
but not in others.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2007-04-03 19:24:02 +08:00
|
|
|
spu_flush_all_slbs(mm);
|
|
|
|
#endif
|
2008-06-18 13:29:12 +08:00
|
|
|
if (get_paca_psize(addr) != MMU_PAGE_4K) {
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
get_paca()->context = mm->context;
|
|
|
|
slb_flush_and_rebolt();
|
|
|
|
}
|
[POWERPC] Allow drivers to map individual 4k pages to userspace
Some drivers have resources that they want to be able to map into
userspace that are 4k in size. On a kernel configured with 64k pages
we currently end up mapping the 4k we want plus another 60k of
physical address space, which could contain anything. This can
introduce security problems, for example in the case of an infiniband
adaptor where the other 60k could contain registers that some other
program is using for its communications.
This patch adds a new function, remap_4k_pfn, which drivers can use to
map a single 4k page to userspace regardless of whether the kernel is
using a 4k or a 64k page size. Like remap_pfn_range, it would
typically be called in a driver's mmap function. It only maps a
single 4k page, which on a 64k page kernel appears replicated 16 times
throughout a 64k page. On a 4k page kernel it reduces to a call to
remap_pfn_range.
The way this works on a 64k kernel is that a new bit, _PAGE_4K_PFN,
gets set on the linux PTE. This alters the way that __hash_page_4K
computes the real address to put in the HPTE. The RPN field of the
linux PTE becomes the 4k RPN directly rather than being interpreted as
a 64k RPN. Since the RPN field is 32 bits, this means that physical
addresses being mapped with remap_4k_pfn have to be below 2^44,
i.e. 0x100000000000.
The patch also factors out the code in arch/powerpc/mm/hash_utils_64.c
that deals with demoting a process to use 4k pages into one function
that gets called in the various different places where we need to do
that. There were some discrepancies between exactly what was done in
the various places, such as a call to spu_flush_all_slbs in one case
but not in others.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2007-04-03 19:24:02 +08:00
|
|
|
}
|
2007-05-08 14:27:27 +08:00
|
|
|
#endif /* CONFIG_PPC_64K_PAGES */
|
[POWERPC] Allow drivers to map individual 4k pages to userspace
Some drivers have resources that they want to be able to map into
userspace that are 4k in size. On a kernel configured with 64k pages
we currently end up mapping the 4k we want plus another 60k of
physical address space, which could contain anything. This can
introduce security problems, for example in the case of an infiniband
adaptor where the other 60k could contain registers that some other
program is using for its communications.
This patch adds a new function, remap_4k_pfn, which drivers can use to
map a single 4k page to userspace regardless of whether the kernel is
using a 4k or a 64k page size. Like remap_pfn_range, it would
typically be called in a driver's mmap function. It only maps a
single 4k page, which on a 64k page kernel appears replicated 16 times
throughout a 64k page. On a 4k page kernel it reduces to a call to
remap_pfn_range.
The way this works on a 64k kernel is that a new bit, _PAGE_4K_PFN,
gets set on the linux PTE. This alters the way that __hash_page_4K
computes the real address to put in the HPTE. The RPN field of the
linux PTE becomes the 4k RPN directly rather than being interpreted as
a 64k RPN. Since the RPN field is 32 bits, this means that physical
addresses being mapped with remap_4k_pfn have to be below 2^44,
i.e. 0x100000000000.
The patch also factors out the code in arch/powerpc/mm/hash_utils_64.c
that deals with demoting a process to use 4k pages into one function
that gets called in the various different places where we need to do
that. There were some discrepancies between exactly what was done in
the various places, such as a call to spu_flush_all_slbs in one case
but not in others.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2007-04-03 19:24:02 +08:00
|
|
|
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
#ifdef CONFIG_PPC_SUBPAGE_PROT
|
|
|
|
/*
|
|
|
|
* This looks up a 2-bit protection code for a 4k subpage of a 64k page.
|
|
|
|
* Userspace sets the subpage permissions using the subpage_prot system call.
|
|
|
|
*
|
|
|
|
* Result is 0: full permissions, _PAGE_RW: read-only,
|
|
|
|
* _PAGE_USER or _PAGE_USER|_PAGE_RW: no access.
|
|
|
|
*/
|
2009-11-27 02:56:04 +08:00
|
|
|
static int subpage_protection(struct mm_struct *mm, unsigned long ea)
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
{
|
2009-11-27 02:56:04 +08:00
|
|
|
struct subpage_prot_table *spt = &mm->context.spt;
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
u32 spp = 0;
|
|
|
|
u32 **sbpm, *sbpp;
|
|
|
|
|
|
|
|
if (ea >= spt->maxaddr)
|
|
|
|
return 0;
|
2013-08-07 00:01:24 +08:00
|
|
|
if (ea < 0x100000000UL) {
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
/* addresses below 4GB use spt->low_prot */
|
|
|
|
sbpm = spt->low_prot;
|
|
|
|
} else {
|
|
|
|
sbpm = spt->protptrs[ea >> SBP_L3_SHIFT];
|
|
|
|
if (!sbpm)
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
sbpp = sbpm[(ea >> SBP_L2_SHIFT) & (SBP_L2_COUNT - 1)];
|
|
|
|
if (!sbpp)
|
|
|
|
return 0;
|
|
|
|
spp = sbpp[(ea >> PAGE_SHIFT) & (SBP_L1_COUNT - 1)];
|
|
|
|
|
|
|
|
/* extract 2-bit bitfield for this 4k subpage */
|
|
|
|
spp >>= 30 - 2 * ((ea >> 12) & 0xf);
|
|
|
|
|
|
|
|
/* turn 0,1,2,3 into combination of _PAGE_USER and _PAGE_RW */
|
|
|
|
spp = ((spp & 2) ? _PAGE_USER : 0) | ((spp & 1) ? _PAGE_RW : 0);
|
|
|
|
return spp;
|
|
|
|
}
|
|
|
|
|
|
|
|
#else /* CONFIG_PPC_SUBPAGE_PROT */
|
2009-11-27 02:56:04 +08:00
|
|
|
static inline int subpage_protection(struct mm_struct *mm, unsigned long ea)
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2010-07-23 08:31:13 +08:00
|
|
|
void hash_failure_debug(unsigned long ea, unsigned long access,
|
|
|
|
unsigned long vsid, unsigned long trap,
|
2013-04-28 17:37:37 +08:00
|
|
|
int ssize, int psize, int lpsize, unsigned long pte)
|
2010-07-23 08:31:13 +08:00
|
|
|
{
|
|
|
|
if (!printk_ratelimit())
|
|
|
|
return;
|
|
|
|
pr_info("mm: Hashing failure ! EA=0x%lx access=0x%lx current=%s\n",
|
|
|
|
ea, access, current->comm);
|
2013-04-28 17:37:37 +08:00
|
|
|
pr_info(" trap=0x%lx vsid=0x%lx ssize=%d base psize=%d psize %d pte=0x%lx\n",
|
|
|
|
trap, vsid, ssize, psize, lpsize, pte);
|
2010-07-23 08:31:13 +08:00
|
|
|
}
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Result code is:
|
|
|
|
* 0 - handled
|
|
|
|
* 1 - normal page fault
|
|
|
|
* -1 - critical hash insertion error
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
* -2 - access not permitted by subpage protection mechanism
|
2005-04-17 06:20:36 +08:00
|
|
|
*/
|
|
|
|
int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
|
|
|
|
{
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
enum ctx_state prev_state = exception_enter();
|
2009-12-16 22:29:56 +08:00
|
|
|
pgd_t *pgdir;
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned long vsid;
|
|
|
|
struct mm_struct *mm;
|
|
|
|
pte_t *ptep;
|
powerpc/mm: Allow more flexible layouts for hugepage pagetables
Currently each available hugepage size uses a slightly different
pagetable layout: that is, the bottem level table of pointers to
hugepages is a different size, and may branch off from the normal page
tables at a different level. Every hugepage aware path that needs to
walk the pagetables must therefore look up the hugepage size from the
slice info first, and work out the correct way to walk the pagetables
accordingly. Future hardware is likely to add more possible hugepage
sizes, more layout options and more mess.
This patch, therefore reworks the handling of hugepage pagetables to
reduce this complexity. In the new scheme, instead of having to
consult the slice mask, pagetable walking code can check a flag in the
PGD/PUD/PMD entries to see where to branch off to hugepage pagetables,
and the entry also contains the information (eseentially hugepage
shift) necessary to then interpret that table without recourse to the
slice mask. This scheme can be extended neatly to handle multiple
levels of self-describing "special" hugepage pagetables, although for
now we assume only one level exists.
This approach means that only the pagetable allocation path needs to
know how the pagetables should be set out. All other (hugepage)
pagetable walking paths can just interpret the structure as they go.
There already was a flag bit in PGD/PUD/PMD entries for hugepage
directory pointers, but it was only used for debug. We alter that
flag bit to instead be a 0 in the MSB to indicate a hugepage pagetable
pointer (normally it would be 1 since the pointer lies in the linear
mapping). This means that asm pagetable walking can test for (and
punt on) hugepage pointers with the same test that checks for
unpopulated page directory entries (beq becomes bge), since hugepage
pointers will always be positive, and normal pointers always negative.
While we're at it, we get rid of the confusing (and grep defeating)
#defining of hugepte_shift to be the same thing as mmu_huge_psizes.
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2009-10-27 03:24:31 +08:00
|
|
|
unsigned hugeshift;
|
2009-03-16 02:16:43 +08:00
|
|
|
const struct cpumask *tmp;
|
2005-11-07 08:06:55 +08:00
|
|
|
int rc, user_region = 0, local = 0;
|
2007-10-11 18:37:10 +08:00
|
|
|
int psize, ssize;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
|
|
|
|
ea, access, trap);
|
2005-05-06 07:15:13 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Get region & vsid */
|
2005-04-17 06:20:36 +08:00
|
|
|
switch (REGION_ID(ea)) {
|
|
|
|
case USER_REGION_ID:
|
|
|
|
user_region = 1;
|
|
|
|
mm = current->mm;
|
2005-11-07 08:06:55 +08:00
|
|
|
if (! mm) {
|
|
|
|
DBG_LOW(" user region with no mm !\n");
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
rc = 1;
|
|
|
|
goto bail;
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
2007-05-08 14:27:28 +08:00
|
|
|
psize = get_slice_psize(mm, ea);
|
2007-10-11 18:37:10 +08:00
|
|
|
ssize = user_segment_size(ea);
|
|
|
|
vsid = get_vsid(mm->context.id, ea, ssize);
|
2005-04-17 06:20:36 +08:00
|
|
|
break;
|
|
|
|
case VMALLOC_REGION_ID:
|
|
|
|
mm = &init_mm;
|
2007-10-11 18:37:10 +08:00
|
|
|
vsid = get_kernel_vsid(ea, mmu_kernel_ssize);
|
2006-06-15 08:45:18 +08:00
|
|
|
if (ea < VMALLOC_END)
|
|
|
|
psize = mmu_vmalloc_psize;
|
|
|
|
else
|
|
|
|
psize = mmu_io_psize;
|
2007-10-11 18:37:10 +08:00
|
|
|
ssize = mmu_kernel_ssize;
|
2005-04-17 06:20:36 +08:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
/* Not a valid range
|
|
|
|
* Send the problem up to do_page_fault
|
|
|
|
*/
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
rc = 1;
|
|
|
|
goto bail;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2005-11-07 08:06:55 +08:00
|
|
|
DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2013-03-13 11:34:54 +08:00
|
|
|
/* Bad address. */
|
|
|
|
if (!vsid) {
|
|
|
|
DBG_LOW("Bad address!\n");
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
rc = 1;
|
|
|
|
goto bail;
|
2013-03-13 11:34:54 +08:00
|
|
|
}
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Get pgdir */
|
2005-04-17 06:20:36 +08:00
|
|
|
pgdir = mm->pgd;
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
if (pgdir == NULL) {
|
|
|
|
rc = 1;
|
|
|
|
goto bail;
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Check CPU locality */
|
2009-03-16 02:16:43 +08:00
|
|
|
tmp = cpumask_of(smp_processor_id());
|
|
|
|
if (user_region && cpumask_equal(mm_cpumask(mm), tmp))
|
2005-04-17 06:20:36 +08:00
|
|
|
local = 1;
|
|
|
|
|
2007-05-08 14:27:28 +08:00
|
|
|
#ifndef CONFIG_PPC_64K_PAGES
|
powerpc/mm: Allow more flexible layouts for hugepage pagetables
Currently each available hugepage size uses a slightly different
pagetable layout: that is, the bottem level table of pointers to
hugepages is a different size, and may branch off from the normal page
tables at a different level. Every hugepage aware path that needs to
walk the pagetables must therefore look up the hugepage size from the
slice info first, and work out the correct way to walk the pagetables
accordingly. Future hardware is likely to add more possible hugepage
sizes, more layout options and more mess.
This patch, therefore reworks the handling of hugepage pagetables to
reduce this complexity. In the new scheme, instead of having to
consult the slice mask, pagetable walking code can check a flag in the
PGD/PUD/PMD entries to see where to branch off to hugepage pagetables,
and the entry also contains the information (eseentially hugepage
shift) necessary to then interpret that table without recourse to the
slice mask. This scheme can be extended neatly to handle multiple
levels of self-describing "special" hugepage pagetables, although for
now we assume only one level exists.
This approach means that only the pagetable allocation path needs to
know how the pagetables should be set out. All other (hugepage)
pagetable walking paths can just interpret the structure as they go.
There already was a flag bit in PGD/PUD/PMD entries for hugepage
directory pointers, but it was only used for debug. We alter that
flag bit to instead be a 0 in the MSB to indicate a hugepage pagetable
pointer (normally it would be 1 since the pointer lies in the linear
mapping). This means that asm pagetable walking can test for (and
punt on) hugepage pointers with the same test that checks for
unpopulated page directory entries (beq becomes bge), since hugepage
pointers will always be positive, and normal pointers always negative.
While we're at it, we get rid of the confusing (and grep defeating)
#defining of hugepte_shift to be the same thing as mmu_huge_psizes.
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2009-10-27 03:24:31 +08:00
|
|
|
/* If we use 4K pages and our psize is not 4K, then we might
|
|
|
|
* be hitting a special driver mapping, and need to align the
|
|
|
|
* address before we fetch the PTE.
|
|
|
|
*
|
|
|
|
* It could also be a hugepage mapping, in which case this is
|
|
|
|
* not necessary, but it's not harmful, either.
|
2007-05-08 14:27:28 +08:00
|
|
|
*/
|
|
|
|
if (psize != MMU_PAGE_4K)
|
|
|
|
ea &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
|
|
|
|
#endif /* CONFIG_PPC_64K_PAGES */
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Get PTE and page size from page tables */
|
powerpc/mm: Allow more flexible layouts for hugepage pagetables
Currently each available hugepage size uses a slightly different
pagetable layout: that is, the bottem level table of pointers to
hugepages is a different size, and may branch off from the normal page
tables at a different level. Every hugepage aware path that needs to
walk the pagetables must therefore look up the hugepage size from the
slice info first, and work out the correct way to walk the pagetables
accordingly. Future hardware is likely to add more possible hugepage
sizes, more layout options and more mess.
This patch, therefore reworks the handling of hugepage pagetables to
reduce this complexity. In the new scheme, instead of having to
consult the slice mask, pagetable walking code can check a flag in the
PGD/PUD/PMD entries to see where to branch off to hugepage pagetables,
and the entry also contains the information (eseentially hugepage
shift) necessary to then interpret that table without recourse to the
slice mask. This scheme can be extended neatly to handle multiple
levels of self-describing "special" hugepage pagetables, although for
now we assume only one level exists.
This approach means that only the pagetable allocation path needs to
know how the pagetables should be set out. All other (hugepage)
pagetable walking paths can just interpret the structure as they go.
There already was a flag bit in PGD/PUD/PMD entries for hugepage
directory pointers, but it was only used for debug. We alter that
flag bit to instead be a 0 in the MSB to indicate a hugepage pagetable
pointer (normally it would be 1 since the pointer lies in the linear
mapping). This means that asm pagetable walking can test for (and
punt on) hugepage pointers with the same test that checks for
unpopulated page directory entries (beq becomes bge), since hugepage
pointers will always be positive, and normal pointers always negative.
While we're at it, we get rid of the confusing (and grep defeating)
#defining of hugepte_shift to be the same thing as mmu_huge_psizes.
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2009-10-27 03:24:31 +08:00
|
|
|
ptep = find_linux_pte_or_hugepte(pgdir, ea, &hugeshift);
|
2005-11-07 08:06:55 +08:00
|
|
|
if (ptep == NULL || !pte_present(*ptep)) {
|
|
|
|
DBG_LOW(" no PTE !\n");
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
rc = 1;
|
|
|
|
goto bail;
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
|
|
|
|
2010-07-23 06:53:23 +08:00
|
|
|
/* Add _PAGE_PRESENT to the required access perm */
|
|
|
|
access |= _PAGE_PRESENT;
|
|
|
|
|
|
|
|
/* Pre-check access permissions (will be re-checked atomically
|
|
|
|
* in __hash_page_XX but this pre-check is a fast path
|
|
|
|
*/
|
|
|
|
if (access & ~pte_val(*ptep)) {
|
|
|
|
DBG_LOW(" no access !\n");
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
rc = 1;
|
|
|
|
goto bail;
|
2010-07-23 06:53:23 +08:00
|
|
|
}
|
|
|
|
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
if (hugeshift) {
|
2013-06-20 17:00:21 +08:00
|
|
|
if (pmd_trans_huge(*(pmd_t *)ptep))
|
|
|
|
rc = __hash_page_thp(ea, access, vsid, (pmd_t *)ptep,
|
|
|
|
trap, local, ssize, psize);
|
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
|
|
else
|
|
|
|
rc = __hash_page_huge(ea, access, vsid, ptep, trap,
|
|
|
|
local, ssize, hugeshift, psize);
|
|
|
|
#else
|
|
|
|
else {
|
|
|
|
/*
|
|
|
|
* if we have hugeshift, and is not transhuge with
|
|
|
|
* hugetlb disabled, something is really wrong.
|
|
|
|
*/
|
|
|
|
rc = 1;
|
|
|
|
WARN_ON(1);
|
|
|
|
}
|
|
|
|
#endif
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
goto bail;
|
|
|
|
}
|
powerpc/mm: Allow more flexible layouts for hugepage pagetables
Currently each available hugepage size uses a slightly different
pagetable layout: that is, the bottem level table of pointers to
hugepages is a different size, and may branch off from the normal page
tables at a different level. Every hugepage aware path that needs to
walk the pagetables must therefore look up the hugepage size from the
slice info first, and work out the correct way to walk the pagetables
accordingly. Future hardware is likely to add more possible hugepage
sizes, more layout options and more mess.
This patch, therefore reworks the handling of hugepage pagetables to
reduce this complexity. In the new scheme, instead of having to
consult the slice mask, pagetable walking code can check a flag in the
PGD/PUD/PMD entries to see where to branch off to hugepage pagetables,
and the entry also contains the information (eseentially hugepage
shift) necessary to then interpret that table without recourse to the
slice mask. This scheme can be extended neatly to handle multiple
levels of self-describing "special" hugepage pagetables, although for
now we assume only one level exists.
This approach means that only the pagetable allocation path needs to
know how the pagetables should be set out. All other (hugepage)
pagetable walking paths can just interpret the structure as they go.
There already was a flag bit in PGD/PUD/PMD entries for hugepage
directory pointers, but it was only used for debug. We alter that
flag bit to instead be a 0 in the MSB to indicate a hugepage pagetable
pointer (normally it would be 1 since the pointer lies in the linear
mapping). This means that asm pagetable walking can test for (and
punt on) hugepage pointers with the same test that checks for
unpopulated page directory entries (beq becomes bge), since hugepage
pointers will always be positive, and normal pointers always negative.
While we're at it, we get rid of the confusing (and grep defeating)
#defining of hugepte_shift to be the same thing as mmu_huge_psizes.
Signed-off-by: David Gibson <dwg@au1.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2009-10-27 03:24:31 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
#ifndef CONFIG_PPC_64K_PAGES
|
|
|
|
DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep));
|
|
|
|
#else
|
|
|
|
DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep),
|
|
|
|
pte_val(*(ptep + PTRS_PER_PTE)));
|
|
|
|
#endif
|
|
|
|
/* Do actual hashing */
|
2007-05-08 14:27:28 +08:00
|
|
|
#ifdef CONFIG_PPC_64K_PAGES
|
[POWERPC] Allow drivers to map individual 4k pages to userspace
Some drivers have resources that they want to be able to map into
userspace that are 4k in size. On a kernel configured with 64k pages
we currently end up mapping the 4k we want plus another 60k of
physical address space, which could contain anything. This can
introduce security problems, for example in the case of an infiniband
adaptor where the other 60k could contain registers that some other
program is using for its communications.
This patch adds a new function, remap_4k_pfn, which drivers can use to
map a single 4k page to userspace regardless of whether the kernel is
using a 4k or a 64k page size. Like remap_pfn_range, it would
typically be called in a driver's mmap function. It only maps a
single 4k page, which on a 64k page kernel appears replicated 16 times
throughout a 64k page. On a 4k page kernel it reduces to a call to
remap_pfn_range.
The way this works on a 64k kernel is that a new bit, _PAGE_4K_PFN,
gets set on the linux PTE. This alters the way that __hash_page_4K
computes the real address to put in the HPTE. The RPN field of the
linux PTE becomes the 4k RPN directly rather than being interpreted as
a 64k RPN. Since the RPN field is 32 bits, this means that physical
addresses being mapped with remap_4k_pfn have to be below 2^44,
i.e. 0x100000000000.
The patch also factors out the code in arch/powerpc/mm/hash_utils_64.c
that deals with demoting a process to use 4k pages into one function
that gets called in the various different places where we need to do
that. There were some discrepancies between exactly what was done in
the various places, such as a call to spu_flush_all_slbs in one case
but not in others.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2007-04-03 19:24:02 +08:00
|
|
|
/* If _PAGE_4K_PFN is set, make sure this is a 4k segment */
|
2008-06-18 13:29:12 +08:00
|
|
|
if ((pte_val(*ptep) & _PAGE_4K_PFN) && psize == MMU_PAGE_64K) {
|
[POWERPC] Allow drivers to map individual 4k pages to userspace
Some drivers have resources that they want to be able to map into
userspace that are 4k in size. On a kernel configured with 64k pages
we currently end up mapping the 4k we want plus another 60k of
physical address space, which could contain anything. This can
introduce security problems, for example in the case of an infiniband
adaptor where the other 60k could contain registers that some other
program is using for its communications.
This patch adds a new function, remap_4k_pfn, which drivers can use to
map a single 4k page to userspace regardless of whether the kernel is
using a 4k or a 64k page size. Like remap_pfn_range, it would
typically be called in a driver's mmap function. It only maps a
single 4k page, which on a 64k page kernel appears replicated 16 times
throughout a 64k page. On a 4k page kernel it reduces to a call to
remap_pfn_range.
The way this works on a 64k kernel is that a new bit, _PAGE_4K_PFN,
gets set on the linux PTE. This alters the way that __hash_page_4K
computes the real address to put in the HPTE. The RPN field of the
linux PTE becomes the 4k RPN directly rather than being interpreted as
a 64k RPN. Since the RPN field is 32 bits, this means that physical
addresses being mapped with remap_4k_pfn have to be below 2^44,
i.e. 0x100000000000.
The patch also factors out the code in arch/powerpc/mm/hash_utils_64.c
that deals with demoting a process to use 4k pages into one function
that gets called in the various different places where we need to do
that. There were some discrepancies between exactly what was done in
the various places, such as a call to spu_flush_all_slbs in one case
but not in others.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2007-04-03 19:24:02 +08:00
|
|
|
demote_segment_4k(mm, ea);
|
|
|
|
psize = MMU_PAGE_4K;
|
|
|
|
}
|
|
|
|
|
2007-05-08 14:27:27 +08:00
|
|
|
/* If this PTE is non-cacheable and we have restrictions on
|
|
|
|
* using non cacheable large pages, then we switch to 4k
|
|
|
|
*/
|
|
|
|
if (mmu_ci_restrictions && psize == MMU_PAGE_64K &&
|
|
|
|
(pte_val(*ptep) & _PAGE_NO_CACHE)) {
|
|
|
|
if (user_region) {
|
|
|
|
demote_segment_4k(mm, ea);
|
|
|
|
psize = MMU_PAGE_4K;
|
|
|
|
} else if (ea < VMALLOC_END) {
|
|
|
|
/*
|
|
|
|
* some driver did a non-cacheable mapping
|
|
|
|
* in vmalloc space, so switch vmalloc
|
|
|
|
* to 4k pages
|
|
|
|
*/
|
|
|
|
printk(KERN_ALERT "Reducing vmalloc segment "
|
|
|
|
"to 4kB pages because of "
|
|
|
|
"non-cacheable mapping\n");
|
|
|
|
psize = mmu_vmalloc_psize = MMU_PAGE_4K;
|
2007-07-17 00:35:38 +08:00
|
|
|
#ifdef CONFIG_SPU_BASE
|
2007-03-10 07:05:37 +08:00
|
|
|
spu_flush_all_slbs(mm);
|
|
|
|
#endif
|
2006-06-15 08:45:18 +08:00
|
|
|
}
|
2007-05-08 14:27:27 +08:00
|
|
|
}
|
|
|
|
if (user_region) {
|
2008-06-18 13:29:12 +08:00
|
|
|
if (psize != get_paca_psize(ea)) {
|
2007-10-29 09:05:18 +08:00
|
|
|
get_paca()->context = mm->context;
|
2006-06-15 08:45:18 +08:00
|
|
|
slb_flush_and_rebolt();
|
|
|
|
}
|
2007-05-08 14:27:27 +08:00
|
|
|
} else if (get_paca()->vmalloc_sllp !=
|
|
|
|
mmu_psize_defs[mmu_vmalloc_psize].sllp) {
|
|
|
|
get_paca()->vmalloc_sllp =
|
|
|
|
mmu_psize_defs[mmu_vmalloc_psize].sllp;
|
2007-08-03 09:55:39 +08:00
|
|
|
slb_vmalloc_update();
|
2006-06-15 08:45:18 +08:00
|
|
|
}
|
2007-05-08 14:27:28 +08:00
|
|
|
#endif /* CONFIG_PPC_64K_PAGES */
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2007-05-08 14:27:28 +08:00
|
|
|
#ifdef CONFIG_PPC_HAS_HASH_64K
|
2006-06-15 08:45:18 +08:00
|
|
|
if (psize == MMU_PAGE_64K)
|
2007-10-11 18:37:10 +08:00
|
|
|
rc = __hash_page_64K(ea, access, vsid, ptep, trap, local, ssize);
|
2005-11-07 08:06:55 +08:00
|
|
|
else
|
2007-05-08 14:27:28 +08:00
|
|
|
#endif /* CONFIG_PPC_HAS_HASH_64K */
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
{
|
2009-12-16 22:29:56 +08:00
|
|
|
int spp = subpage_protection(mm, ea);
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
if (access & spp)
|
|
|
|
rc = -2;
|
|
|
|
else
|
|
|
|
rc = __hash_page_4K(ea, access, vsid, ptep, trap,
|
|
|
|
local, ssize, spp);
|
|
|
|
}
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2010-07-23 08:31:13 +08:00
|
|
|
/* Dump some info in case of hash insertion failure, they should
|
|
|
|
* never happen so it is really useful to know if/when they do
|
|
|
|
*/
|
|
|
|
if (rc == -1)
|
|
|
|
hash_failure_debug(ea, access, vsid, trap, ssize, psize,
|
2013-04-28 17:37:37 +08:00
|
|
|
psize, pte_val(*ptep));
|
2005-11-07 08:06:55 +08:00
|
|
|
#ifndef CONFIG_PPC_64K_PAGES
|
|
|
|
DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep));
|
|
|
|
#else
|
|
|
|
DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep),
|
|
|
|
pte_val(*(ptep + PTRS_PER_PTE)));
|
|
|
|
#endif
|
|
|
|
DBG_LOW(" -> rc=%d\n", rc);
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
|
|
|
|
bail:
|
|
|
|
exception_exit(prev_state);
|
2005-11-07 08:06:55 +08:00
|
|
|
return rc;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2005-11-16 04:53:48 +08:00
|
|
|
EXPORT_SYMBOL_GPL(hash_page);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
void hash_preload(struct mm_struct *mm, unsigned long ea,
|
|
|
|
unsigned long access, unsigned long trap)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2013-06-20 17:00:18 +08:00
|
|
|
int hugepage_shift;
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned long vsid;
|
2010-11-18 02:52:45 +08:00
|
|
|
pgd_t *pgdir;
|
2005-11-07 08:06:55 +08:00
|
|
|
pte_t *ptep;
|
|
|
|
unsigned long flags;
|
2010-07-23 08:31:13 +08:00
|
|
|
int rc, ssize, local = 0;
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2007-05-08 14:27:27 +08:00
|
|
|
BUG_ON(REGION_ID(ea) != USER_REGION_ID);
|
|
|
|
|
|
|
|
#ifdef CONFIG_PPC_MM_SLICES
|
|
|
|
/* We only prefault standard pages for now */
|
2007-08-16 06:03:35 +08:00
|
|
|
if (unlikely(get_slice_psize(mm, ea) != mm->context.user_psize))
|
2005-11-07 08:06:55 +08:00
|
|
|
return;
|
2007-05-08 14:27:27 +08:00
|
|
|
#endif
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
DBG_LOW("hash_preload(mm=%p, mm->pgdir=%p, ea=%016lx, access=%lx,"
|
|
|
|
" trap=%lx\n", mm, mm->pgd, ea, access, trap);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-05-08 14:27:27 +08:00
|
|
|
/* Get Linux PTE if available */
|
2005-11-07 08:06:55 +08:00
|
|
|
pgdir = mm->pgd;
|
|
|
|
if (pgdir == NULL)
|
|
|
|
return;
|
2013-06-20 17:00:22 +08:00
|
|
|
|
|
|
|
/* Get VSID */
|
|
|
|
ssize = user_segment_size(ea);
|
|
|
|
vsid = get_vsid(mm->context.id, ea, ssize);
|
|
|
|
if (!vsid)
|
|
|
|
return;
|
|
|
|
/*
|
|
|
|
* Hash doesn't like irqs. Walking linux page table with irq disabled
|
|
|
|
* saves us from holding multiple locks.
|
|
|
|
*/
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
2013-06-20 17:00:18 +08:00
|
|
|
/*
|
|
|
|
* THP pages use update_mmu_cache_pmd. We don't do
|
|
|
|
* hash preload there. Hence can ignore THP here
|
|
|
|
*/
|
|
|
|
ptep = find_linux_pte_or_hugepte(pgdir, ea, &hugepage_shift);
|
2005-11-07 08:06:55 +08:00
|
|
|
if (!ptep)
|
2013-06-20 17:00:22 +08:00
|
|
|
goto out_exit;
|
2007-05-08 14:27:27 +08:00
|
|
|
|
2013-06-20 17:00:18 +08:00
|
|
|
WARN_ON(hugepage_shift);
|
2007-05-08 14:27:27 +08:00
|
|
|
#ifdef CONFIG_PPC_64K_PAGES
|
|
|
|
/* If either _PAGE_4K_PFN or _PAGE_NO_CACHE is set (and we are on
|
|
|
|
* a 64K kernel), then we don't preload, hash_page() will take
|
|
|
|
* care of it once we actually try to access the page.
|
|
|
|
* That way we don't have to duplicate all of the logic for segment
|
|
|
|
* page size demotion here
|
|
|
|
*/
|
|
|
|
if (pte_val(*ptep) & (_PAGE_4K_PFN | _PAGE_NO_CACHE))
|
2013-06-20 17:00:22 +08:00
|
|
|
goto out_exit;
|
2007-05-08 14:27:27 +08:00
|
|
|
#endif /* CONFIG_PPC_64K_PAGES */
|
|
|
|
|
2007-05-08 14:27:28 +08:00
|
|
|
/* Is that local to this CPU ? */
|
2009-03-16 02:16:43 +08:00
|
|
|
if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
|
2005-11-07 08:06:55 +08:00
|
|
|
local = 1;
|
2007-05-08 14:27:28 +08:00
|
|
|
|
|
|
|
/* Hash it in */
|
|
|
|
#ifdef CONFIG_PPC_HAS_HASH_64K
|
2006-06-15 08:45:18 +08:00
|
|
|
if (mm->context.user_psize == MMU_PAGE_64K)
|
2010-07-23 08:31:13 +08:00
|
|
|
rc = __hash_page_64K(ea, access, vsid, ptep, trap, local, ssize);
|
2005-04-17 06:20:36 +08:00
|
|
|
else
|
2007-05-17 02:43:02 +08:00
|
|
|
#endif /* CONFIG_PPC_HAS_HASH_64K */
|
2010-07-23 08:31:13 +08:00
|
|
|
rc = __hash_page_4K(ea, access, vsid, ptep, trap, local, ssize,
|
2010-11-18 00:32:59 +08:00
|
|
|
subpage_protection(mm, ea));
|
2010-07-23 08:31:13 +08:00
|
|
|
|
|
|
|
/* Dump some info in case of hash insertion failure, they should
|
|
|
|
* never happen so it is really useful to know if/when they do
|
|
|
|
*/
|
|
|
|
if (rc == -1)
|
|
|
|
hash_failure_debug(ea, access, vsid, trap, ssize,
|
2013-04-28 17:37:37 +08:00
|
|
|
mm->context.user_psize,
|
|
|
|
mm->context.user_psize,
|
|
|
|
pte_val(*ptep));
|
2013-06-20 17:00:22 +08:00
|
|
|
out_exit:
|
2005-11-07 08:06:55 +08:00
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
|
2007-10-29 09:05:18 +08:00
|
|
|
/* WARNING: This is called from hash_low_64.S, if you change this prototype,
|
|
|
|
* do not forget to update the assembly call site !
|
|
|
|
*/
|
2012-09-10 10:52:50 +08:00
|
|
|
void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
|
2007-10-11 18:37:10 +08:00
|
|
|
int local)
|
2005-11-07 08:06:55 +08:00
|
|
|
{
|
|
|
|
unsigned long hash, index, shift, hidx, slot;
|
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
DBG_LOW("flush_hash_page(vpn=%016lx)\n", vpn);
|
|
|
|
pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
|
|
|
|
hash = hpt_hash(vpn, shift, ssize);
|
2005-11-07 08:06:55 +08:00
|
|
|
hidx = __rpte_to_hidx(pte, index);
|
|
|
|
if (hidx & _PTEIDX_SECONDARY)
|
|
|
|
hash = ~hash;
|
|
|
|
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
|
|
|
|
slot += hidx & _PTEIDX_GROUP_IX;
|
2009-12-14 05:15:12 +08:00
|
|
|
DBG_LOW(" sub %ld: hash=%lx, hidx=%lx\n", index, slot, hidx);
|
2013-06-20 17:00:13 +08:00
|
|
|
/*
|
|
|
|
* We use same base page size and actual psize, because we don't
|
|
|
|
* use these functions for hugepage
|
|
|
|
*/
|
|
|
|
ppc_md.hpte_invalidate(slot, vpn, psize, psize, ssize, local);
|
2005-11-07 08:06:55 +08:00
|
|
|
} pte_iterate_hashed_end();
|
2013-02-14 00:21:40 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
|
|
|
|
/* Transactions are not aborted by tlbiel, only tlbie.
|
|
|
|
* Without, syncing a page back to a block device w/ PIO could pick up
|
|
|
|
* transactional data (bad!) so we force an abort here. Before the
|
|
|
|
* sync the page will be made read-only, which will flush_hash_page.
|
|
|
|
* BIG ISSUE here: if the kernel uses a page from userspace without
|
|
|
|
* unmapping it first, it may see the speculated version.
|
|
|
|
*/
|
|
|
|
if (local && cpu_has_feature(CPU_FTR_TM) &&
|
2013-05-02 23:36:14 +08:00
|
|
|
current->thread.regs &&
|
2013-02-14 00:21:40 +08:00
|
|
|
MSR_TM_ACTIVE(current->thread.regs->msr)) {
|
|
|
|
tm_enable();
|
|
|
|
tm_abort(TM_CAUSE_TLBI);
|
|
|
|
}
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
2005-09-20 11:52:50 +08:00
|
|
|
void flush_hash_range(unsigned long number, int local)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2005-11-07 08:06:55 +08:00
|
|
|
if (ppc_md.flush_hash_range)
|
2005-09-20 11:52:50 +08:00
|
|
|
ppc_md.flush_hash_range(number, local);
|
2005-11-07 08:06:55 +08:00
|
|
|
else {
|
2005-04-17 06:20:36 +08:00
|
|
|
int i;
|
2005-09-20 11:52:50 +08:00
|
|
|
struct ppc64_tlb_batch *batch =
|
|
|
|
&__get_cpu_var(ppc64_tlb_batch);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
for (i = 0; i < number; i++)
|
2012-09-10 10:52:50 +08:00
|
|
|
flush_hash_page(batch->vpn[i], batch->pte[i],
|
2007-10-11 18:37:10 +08:00
|
|
|
batch->psize, batch->ssize, local);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* low_hash_fault is called when we the low level hash code failed
|
|
|
|
* to instert a PTE due to an hypervisor error
|
|
|
|
*/
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
enum ctx_state prev_state = exception_enter();
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
if (user_mode(regs)) {
|
[POWERPC] Provide a way to protect 4k subpages when using 64k pages
Using 64k pages on 64-bit PowerPC systems makes life difficult for
emulators that are trying to emulate an ISA, such as x86, which use a
smaller page size, since the emulator can no longer use the MMU and
the normal system calls for controlling page protections. Of course,
the emulator can emulate the MMU by checking and possibly remapping
the address for each memory access in software, but that is pretty
slow.
This provides a facility for such programs to control the access
permissions on individual 4k sub-pages of 64k pages. The idea is
that the emulator supplies an array of protection masks to apply to a
specified range of virtual addresses. These masks are applied at the
level where hardware PTEs are inserted into the hardware page table
based on the Linux PTEs, so the Linux PTEs are not affected. Note
that this new mechanism does not allow any access that would otherwise
be prohibited; it can only prohibit accesses that would otherwise be
allowed. This new facility is only available on 64-bit PowerPC and
only when the kernel is configured for 64k pages.
The masks are supplied using a new subpage_prot system call, which
takes a starting virtual address and length, and a pointer to an array
of protection masks in memory. The array has a 32-bit word per 64k
page to be protected; each 32-bit word consists of 16 2-bit fields,
for which 0 allows any access (that is otherwise allowed), 1 prevents
write accesses, and 2 or 3 prevent any access.
Implicit in this is that the regions of the address space that are
protected are switched to use 4k hardware pages rather than 64k
hardware pages (on machines with hardware 64k page support). In fact
the whole process is switched to use 4k hardware pages when the
subpage_prot system call is used, but this could be improved in future
to switch only the affected segments.
The subpage protection bits are stored in a 3 level tree akin to the
page table tree. The top level of this tree is stored in a structure
that is appended to the top level of the page table tree, i.e., the
pgd array. Since it will often only be 32-bit addresses (below 4GB)
that are protected, the pointers to the first four bottom level pages
are also stored in this structure (each bottom level page contains the
protection bits for 1GB of address space), so the protection bits for
addresses below 4GB can be accessed with one fewer loads than those
for higher addresses.
Signed-off-by: Paul Mackerras <paulus@samba.org>
2008-01-24 05:35:13 +08:00
|
|
|
#ifdef CONFIG_PPC_SUBPAGE_PROT
|
|
|
|
if (rc == -2)
|
|
|
|
_exception(SIGSEGV, regs, SEGV_ACCERR, address);
|
|
|
|
else
|
|
|
|
#endif
|
|
|
|
_exception(SIGBUS, regs, BUS_ADRERR, address);
|
|
|
|
} else
|
|
|
|
bad_page_fault(regs, address, SIGBUS);
|
powerpc: Exception hooks for context tracking subsystem
This is the exception hooks for context tracking subsystem, including
data access, program check, single step, instruction breakpoint, machine check,
alignment, fp unavailable, altivec assist, unknown exception, whose handlers
might use RCU.
This patch corresponds to
[PATCH] x86: Exception hooks for userspace RCU extended QS
commit 6ba3c97a38803883c2eee489505796cb0a727122
But after the exception handling moved to generic code, and some changes in
following two commits:
56dd9470d7c8734f055da2a6bac553caf4a468eb
context_tracking: Move exception handling to generic code
6c1e0256fad84a843d915414e4b5973b7443d48d
context_tracking: Restore correct previous context state on exception exit
it is able for exception hooks to use the generic code above instead of a
redundant arch implementation.
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2013-05-14 00:16:41 +08:00
|
|
|
|
|
|
|
exception_exit(prev_state);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
2007-04-12 13:30:23 +08:00
|
|
|
|
2013-04-16 00:53:19 +08:00
|
|
|
long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
|
|
|
|
unsigned long pa, unsigned long rflags,
|
|
|
|
unsigned long vflags, int psize, int ssize)
|
|
|
|
{
|
|
|
|
unsigned long hpte_group;
|
|
|
|
long slot;
|
|
|
|
|
|
|
|
repeat:
|
|
|
|
hpte_group = ((hash & htab_hash_mask) *
|
|
|
|
HPTES_PER_GROUP) & ~0x7UL;
|
|
|
|
|
|
|
|
/* Insert into the hash table, primary slot */
|
|
|
|
slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, vflags,
|
2013-04-28 17:37:35 +08:00
|
|
|
psize, psize, ssize);
|
2013-04-16 00:53:19 +08:00
|
|
|
|
|
|
|
/* Primary is full, try the secondary */
|
|
|
|
if (unlikely(slot == -1)) {
|
|
|
|
hpte_group = ((~hash & htab_hash_mask) *
|
|
|
|
HPTES_PER_GROUP) & ~0x7UL;
|
|
|
|
slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags,
|
|
|
|
vflags | HPTE_V_SECONDARY,
|
2013-04-28 17:37:35 +08:00
|
|
|
psize, psize, ssize);
|
2013-04-16 00:53:19 +08:00
|
|
|
if (slot == -1) {
|
|
|
|
if (mftb() & 0x1)
|
|
|
|
hpte_group = ((hash & htab_hash_mask) *
|
|
|
|
HPTES_PER_GROUP)&~0x7UL;
|
|
|
|
|
|
|
|
ppc_md.hpte_remove(hpte_group);
|
|
|
|
goto repeat;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return slot;
|
|
|
|
}
|
|
|
|
|
2007-04-12 13:30:23 +08:00
|
|
|
#ifdef CONFIG_DEBUG_PAGEALLOC
|
|
|
|
static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi)
|
|
|
|
{
|
powerpc: Try to insert the hptes repeatedly in kernel_map_linear_page()
This patch fixes the following oops, which could be trigged by build the kernel
with many concurrent threads, under CONFIG_DEBUG_PAGEALLOC.
hpte_insert() might return -1, indicating that the bucket (primary here)
is full. We are not necessarily reporting a BUG in this case. Instead, we could
try repeatedly (try secondary, remove and try again) until we find a slot.
[ 543.075675] ------------[ cut here ]------------
[ 543.075701] kernel BUG at arch/powerpc/mm/hash_utils_64.c:1239!
[ 543.075714] Oops: Exception in kernel mode, sig: 5 [#1]
[ 543.075722] PREEMPT SMP NR_CPUS=16 DEBUG_PAGEALLOC NUMA pSeries
[ 543.075741] Modules linked in: binfmt_misc ehea
[ 543.075759] NIP: c000000000036eb0 LR: c000000000036ea4 CTR: c00000000005a594
[ 543.075771] REGS: c0000000a90832c0 TRAP: 0700 Not tainted (3.8.0-next-20130222)
[ 543.075781] MSR: 8000000000029032 <SF,EE,ME,IR,DR,RI> CR: 22224482 XER: 00000000
[ 543.075816] SOFTE: 0
[ 543.075823] CFAR: c00000000004c200
[ 543.075830] TASK = c0000000e506b750[23934] 'cc1' THREAD: c0000000a9080000 CPU: 1
GPR00: 0000000000000001 c0000000a9083540 c000000000c600a8 ffffffffffffffff
GPR04: 0000000000000050 fffffffffffffffa c0000000a90834e0 00000000004ff594
GPR08: 0000000000000001 0000000000000000 000000009592d4d8 c000000000c86854
GPR12: 0000000000000002 c000000006ead300 0000000000a51000 0000000000000001
GPR16: f000000003354380 ffffffffffffffff ffffffffffffff80 0000000000000000
GPR20: 0000000000000001 c000000000c600a8 0000000000000001 0000000000000001
GPR24: 0000000003354380 c000000000000000 0000000000000000 c000000000b65950
GPR28: 0000002000000000 00000000000cd50e 0000000000bf50d9 c000000000c7c230
[ 543.076005] NIP [c000000000036eb0] .kernel_map_pages+0x1e0/0x3f8
[ 543.076016] LR [c000000000036ea4] .kernel_map_pages+0x1d4/0x3f8
[ 543.076025] Call Trace:
[ 543.076033] [c0000000a9083540] [c000000000036ea4] .kernel_map_pages+0x1d4/0x3f8 (unreliable)
[ 543.076053] [c0000000a9083640] [c000000000167638] .get_page_from_freelist+0x6cc/0x8dc
[ 543.076067] [c0000000a9083800] [c000000000167a48] .__alloc_pages_nodemask+0x200/0x96c
[ 543.076082] [c0000000a90839c0] [c0000000001ade44] .alloc_pages_vma+0x160/0x1e4
[ 543.076098] [c0000000a9083a80] [c00000000018ce04] .handle_pte_fault+0x1b0/0x7e8
[ 543.076113] [c0000000a9083b50] [c00000000018d5a8] .handle_mm_fault+0x16c/0x1a0
[ 543.076129] [c0000000a9083c00] [c0000000007bf1dc] .do_page_fault+0x4d0/0x7a4
[ 543.076144] [c0000000a9083e30] [c0000000000090e8] handle_page_fault+0x10/0x30
[ 543.076155] Instruction dump:
[ 543.076163] 7c630038 78631d88 e80a0000 f8410028 7c0903a6 e91f01de e96a0010 e84a0008
[ 543.076192] 4e800421 e8410028 7c7107b4 7a200fe0 <0b000000> 7f63db78 48785781 60000000
[ 543.076224] ---[ end trace bd5807e8d6ae186b ]---
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
2013-04-16 00:53:20 +08:00
|
|
|
unsigned long hash;
|
2007-10-11 18:37:10 +08:00
|
|
|
unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize);
|
2012-09-10 10:52:50 +08:00
|
|
|
unsigned long vpn = hpt_vpn(vaddr, vsid, mmu_kernel_ssize);
|
2008-08-05 14:19:56 +08:00
|
|
|
unsigned long mode = htab_convert_pte_flags(PAGE_KERNEL);
|
powerpc: Try to insert the hptes repeatedly in kernel_map_linear_page()
This patch fixes the following oops, which could be trigged by build the kernel
with many concurrent threads, under CONFIG_DEBUG_PAGEALLOC.
hpte_insert() might return -1, indicating that the bucket (primary here)
is full. We are not necessarily reporting a BUG in this case. Instead, we could
try repeatedly (try secondary, remove and try again) until we find a slot.
[ 543.075675] ------------[ cut here ]------------
[ 543.075701] kernel BUG at arch/powerpc/mm/hash_utils_64.c:1239!
[ 543.075714] Oops: Exception in kernel mode, sig: 5 [#1]
[ 543.075722] PREEMPT SMP NR_CPUS=16 DEBUG_PAGEALLOC NUMA pSeries
[ 543.075741] Modules linked in: binfmt_misc ehea
[ 543.075759] NIP: c000000000036eb0 LR: c000000000036ea4 CTR: c00000000005a594
[ 543.075771] REGS: c0000000a90832c0 TRAP: 0700 Not tainted (3.8.0-next-20130222)
[ 543.075781] MSR: 8000000000029032 <SF,EE,ME,IR,DR,RI> CR: 22224482 XER: 00000000
[ 543.075816] SOFTE: 0
[ 543.075823] CFAR: c00000000004c200
[ 543.075830] TASK = c0000000e506b750[23934] 'cc1' THREAD: c0000000a9080000 CPU: 1
GPR00: 0000000000000001 c0000000a9083540 c000000000c600a8 ffffffffffffffff
GPR04: 0000000000000050 fffffffffffffffa c0000000a90834e0 00000000004ff594
GPR08: 0000000000000001 0000000000000000 000000009592d4d8 c000000000c86854
GPR12: 0000000000000002 c000000006ead300 0000000000a51000 0000000000000001
GPR16: f000000003354380 ffffffffffffffff ffffffffffffff80 0000000000000000
GPR20: 0000000000000001 c000000000c600a8 0000000000000001 0000000000000001
GPR24: 0000000003354380 c000000000000000 0000000000000000 c000000000b65950
GPR28: 0000002000000000 00000000000cd50e 0000000000bf50d9 c000000000c7c230
[ 543.076005] NIP [c000000000036eb0] .kernel_map_pages+0x1e0/0x3f8
[ 543.076016] LR [c000000000036ea4] .kernel_map_pages+0x1d4/0x3f8
[ 543.076025] Call Trace:
[ 543.076033] [c0000000a9083540] [c000000000036ea4] .kernel_map_pages+0x1d4/0x3f8 (unreliable)
[ 543.076053] [c0000000a9083640] [c000000000167638] .get_page_from_freelist+0x6cc/0x8dc
[ 543.076067] [c0000000a9083800] [c000000000167a48] .__alloc_pages_nodemask+0x200/0x96c
[ 543.076082] [c0000000a90839c0] [c0000000001ade44] .alloc_pages_vma+0x160/0x1e4
[ 543.076098] [c0000000a9083a80] [c00000000018ce04] .handle_pte_fault+0x1b0/0x7e8
[ 543.076113] [c0000000a9083b50] [c00000000018d5a8] .handle_mm_fault+0x16c/0x1a0
[ 543.076129] [c0000000a9083c00] [c0000000007bf1dc] .do_page_fault+0x4d0/0x7a4
[ 543.076144] [c0000000a9083e30] [c0000000000090e8] handle_page_fault+0x10/0x30
[ 543.076155] Instruction dump:
[ 543.076163] 7c630038 78631d88 e80a0000 f8410028 7c0903a6 e91f01de e96a0010 e84a0008
[ 543.076192] 4e800421 e8410028 7c7107b4 7a200fe0 <0b000000> 7f63db78 48785781 60000000
[ 543.076224] ---[ end trace bd5807e8d6ae186b ]---
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
2013-04-16 00:53:20 +08:00
|
|
|
long ret;
|
2007-04-12 13:30:23 +08:00
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
hash = hpt_hash(vpn, PAGE_SHIFT, mmu_kernel_ssize);
|
2007-04-12 13:30:23 +08:00
|
|
|
|
2013-03-13 11:34:54 +08:00
|
|
|
/* Don't create HPTE entries for bad address */
|
|
|
|
if (!vsid)
|
|
|
|
return;
|
powerpc: Try to insert the hptes repeatedly in kernel_map_linear_page()
This patch fixes the following oops, which could be trigged by build the kernel
with many concurrent threads, under CONFIG_DEBUG_PAGEALLOC.
hpte_insert() might return -1, indicating that the bucket (primary here)
is full. We are not necessarily reporting a BUG in this case. Instead, we could
try repeatedly (try secondary, remove and try again) until we find a slot.
[ 543.075675] ------------[ cut here ]------------
[ 543.075701] kernel BUG at arch/powerpc/mm/hash_utils_64.c:1239!
[ 543.075714] Oops: Exception in kernel mode, sig: 5 [#1]
[ 543.075722] PREEMPT SMP NR_CPUS=16 DEBUG_PAGEALLOC NUMA pSeries
[ 543.075741] Modules linked in: binfmt_misc ehea
[ 543.075759] NIP: c000000000036eb0 LR: c000000000036ea4 CTR: c00000000005a594
[ 543.075771] REGS: c0000000a90832c0 TRAP: 0700 Not tainted (3.8.0-next-20130222)
[ 543.075781] MSR: 8000000000029032 <SF,EE,ME,IR,DR,RI> CR: 22224482 XER: 00000000
[ 543.075816] SOFTE: 0
[ 543.075823] CFAR: c00000000004c200
[ 543.075830] TASK = c0000000e506b750[23934] 'cc1' THREAD: c0000000a9080000 CPU: 1
GPR00: 0000000000000001 c0000000a9083540 c000000000c600a8 ffffffffffffffff
GPR04: 0000000000000050 fffffffffffffffa c0000000a90834e0 00000000004ff594
GPR08: 0000000000000001 0000000000000000 000000009592d4d8 c000000000c86854
GPR12: 0000000000000002 c000000006ead300 0000000000a51000 0000000000000001
GPR16: f000000003354380 ffffffffffffffff ffffffffffffff80 0000000000000000
GPR20: 0000000000000001 c000000000c600a8 0000000000000001 0000000000000001
GPR24: 0000000003354380 c000000000000000 0000000000000000 c000000000b65950
GPR28: 0000002000000000 00000000000cd50e 0000000000bf50d9 c000000000c7c230
[ 543.076005] NIP [c000000000036eb0] .kernel_map_pages+0x1e0/0x3f8
[ 543.076016] LR [c000000000036ea4] .kernel_map_pages+0x1d4/0x3f8
[ 543.076025] Call Trace:
[ 543.076033] [c0000000a9083540] [c000000000036ea4] .kernel_map_pages+0x1d4/0x3f8 (unreliable)
[ 543.076053] [c0000000a9083640] [c000000000167638] .get_page_from_freelist+0x6cc/0x8dc
[ 543.076067] [c0000000a9083800] [c000000000167a48] .__alloc_pages_nodemask+0x200/0x96c
[ 543.076082] [c0000000a90839c0] [c0000000001ade44] .alloc_pages_vma+0x160/0x1e4
[ 543.076098] [c0000000a9083a80] [c00000000018ce04] .handle_pte_fault+0x1b0/0x7e8
[ 543.076113] [c0000000a9083b50] [c00000000018d5a8] .handle_mm_fault+0x16c/0x1a0
[ 543.076129] [c0000000a9083c00] [c0000000007bf1dc] .do_page_fault+0x4d0/0x7a4
[ 543.076144] [c0000000a9083e30] [c0000000000090e8] handle_page_fault+0x10/0x30
[ 543.076155] Instruction dump:
[ 543.076163] 7c630038 78631d88 e80a0000 f8410028 7c0903a6 e91f01de e96a0010 e84a0008
[ 543.076192] 4e800421 e8410028 7c7107b4 7a200fe0 <0b000000> 7f63db78 48785781 60000000
[ 543.076224] ---[ end trace bd5807e8d6ae186b ]---
Signed-off-by: Li Zhong <zhong@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
2013-04-16 00:53:20 +08:00
|
|
|
|
|
|
|
ret = hpte_insert_repeating(hash, vpn, __pa(vaddr), mode,
|
|
|
|
HPTE_V_BOLTED,
|
|
|
|
mmu_linear_psize, mmu_kernel_ssize);
|
|
|
|
|
2007-04-12 13:30:23 +08:00
|
|
|
BUG_ON (ret < 0);
|
|
|
|
spin_lock(&linear_map_hash_lock);
|
|
|
|
BUG_ON(linear_map_hash_slots[lmi] & 0x80);
|
|
|
|
linear_map_hash_slots[lmi] = ret | 0x80;
|
|
|
|
spin_unlock(&linear_map_hash_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
|
|
|
|
{
|
2007-10-11 18:37:10 +08:00
|
|
|
unsigned long hash, hidx, slot;
|
|
|
|
unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize);
|
2012-09-10 10:52:50 +08:00
|
|
|
unsigned long vpn = hpt_vpn(vaddr, vsid, mmu_kernel_ssize);
|
2007-04-12 13:30:23 +08:00
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
hash = hpt_hash(vpn, PAGE_SHIFT, mmu_kernel_ssize);
|
2007-04-12 13:30:23 +08:00
|
|
|
spin_lock(&linear_map_hash_lock);
|
|
|
|
BUG_ON(!(linear_map_hash_slots[lmi] & 0x80));
|
|
|
|
hidx = linear_map_hash_slots[lmi] & 0x7f;
|
|
|
|
linear_map_hash_slots[lmi] = 0;
|
|
|
|
spin_unlock(&linear_map_hash_lock);
|
|
|
|
if (hidx & _PTEIDX_SECONDARY)
|
|
|
|
hash = ~hash;
|
|
|
|
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
|
|
|
|
slot += hidx & _PTEIDX_GROUP_IX;
|
2013-06-20 17:00:13 +08:00
|
|
|
ppc_md.hpte_invalidate(slot, vpn, mmu_linear_psize, mmu_linear_psize,
|
|
|
|
mmu_kernel_ssize, 0);
|
2007-04-12 13:30:23 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void kernel_map_pages(struct page *page, int numpages, int enable)
|
|
|
|
{
|
|
|
|
unsigned long flags, vaddr, lmi;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
for (i = 0; i < numpages; i++, page++) {
|
|
|
|
vaddr = (unsigned long)page_address(page);
|
|
|
|
lmi = __pa(vaddr) >> PAGE_SHIFT;
|
|
|
|
if (lmi >= linear_map_hash_count)
|
|
|
|
continue;
|
|
|
|
if (enable)
|
|
|
|
kernel_map_linear_page(vaddr, lmi);
|
|
|
|
else
|
|
|
|
kernel_unmap_linear_page(vaddr, lmi);
|
|
|
|
}
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_DEBUG_PAGEALLOC */
|
2010-07-07 06:39:02 +08:00
|
|
|
|
|
|
|
void setup_initial_memory_limit(phys_addr_t first_memblock_base,
|
|
|
|
phys_addr_t first_memblock_size)
|
|
|
|
{
|
|
|
|
/* We don't currently support the first MEMBLOCK not mapping 0
|
|
|
|
* physical on those processors
|
|
|
|
*/
|
|
|
|
BUG_ON(first_memblock_base != 0);
|
|
|
|
|
|
|
|
/* On LPAR systems, the first entry is our RMA region,
|
|
|
|
* non-LPAR 64-bit hash MMU systems don't have a limitation
|
|
|
|
* on real mode access, but using the first entry works well
|
|
|
|
* enough. We also clamp it to 1G to avoid some funky things
|
|
|
|
* such as RTAS bugs etc...
|
|
|
|
*/
|
|
|
|
ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
|
|
|
|
|
|
|
|
/* Finally limit subsequent allocations */
|
|
|
|
memblock_set_current_limit(ppc64_rma_size);
|
|
|
|
}
|