2019-05-27 14:55:21 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2013-04-10 20:48:00 +08:00
|
|
|
/*
|
|
|
|
* arch/arm64/mm/hugetlbpage.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 2013 Linaro Ltd.
|
|
|
|
*
|
|
|
|
* Based on arch/x86/mm/hugetlbpage.c.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/hugetlb.h>
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/sysctl.h>
|
|
|
|
#include <asm/mman.h>
|
|
|
|
#include <asm/tlb.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
|
2020-07-01 12:42:01 +08:00
|
|
|
/*
|
|
|
|
* HugeTLB Support Matrix
|
|
|
|
*
|
|
|
|
* ---------------------------------------------------
|
|
|
|
* | Page Size | CONT PTE | PMD | CONT PMD | PUD |
|
|
|
|
* ---------------------------------------------------
|
|
|
|
* | 4K | 64K | 2M | 32M | 1G |
|
|
|
|
* | 16K | 2M | 32M | 1G | |
|
|
|
|
* | 64K | 2M | 512M | 16G | |
|
|
|
|
* ---------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Reserve CMA areas for the largest supported gigantic
|
|
|
|
* huge page when requested. Any other smaller gigantic
|
|
|
|
* huge pages could still be served from those areas.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_CMA
|
|
|
|
void __init arm64_hugetlb_cma_reserve(void)
|
|
|
|
{
|
|
|
|
int order;
|
|
|
|
|
2021-09-20 17:29:31 +08:00
|
|
|
if (pud_sect_supported())
|
|
|
|
order = PUD_SHIFT - PAGE_SHIFT;
|
|
|
|
else
|
2021-10-29 19:27:53 +08:00
|
|
|
order = CONT_PMD_SHIFT - PAGE_SHIFT;
|
|
|
|
|
2020-07-01 12:42:01 +08:00
|
|
|
/*
|
|
|
|
* HugeTLB CMA reservation is required for gigantic
|
|
|
|
* huge pages which could not be allocated via the
|
|
|
|
* page allocator. Just warn if there is any change
|
|
|
|
* breaking this assumption.
|
|
|
|
*/
|
|
|
|
WARN_ON(order <= MAX_ORDER);
|
|
|
|
hugetlb_cma_reserve(order);
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_CMA */
|
|
|
|
|
2022-02-17 12:52:37 +08:00
|
|
|
static bool __hugetlb_valid_size(unsigned long size)
|
2019-03-06 07:43:58 +08:00
|
|
|
{
|
2022-02-17 12:52:37 +08:00
|
|
|
switch (size) {
|
2021-09-20 17:29:31 +08:00
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
2019-03-06 07:43:58 +08:00
|
|
|
case PUD_SIZE:
|
2021-09-20 17:29:31 +08:00
|
|
|
return pud_sect_supported();
|
2019-03-06 07:43:58 +08:00
|
|
|
#endif
|
|
|
|
case CONT_PMD_SIZE:
|
2022-02-17 12:52:37 +08:00
|
|
|
case PMD_SIZE:
|
2019-03-06 07:43:58 +08:00
|
|
|
case CONT_PTE_SIZE:
|
|
|
|
return true;
|
|
|
|
}
|
2022-02-17 12:52:37 +08:00
|
|
|
|
2019-03-06 07:43:58 +08:00
|
|
|
return false;
|
|
|
|
}
|
2022-02-17 12:52:37 +08:00
|
|
|
|
|
|
|
#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
|
|
|
|
bool arch_hugetlb_migration_supported(struct hstate *h)
|
|
|
|
{
|
|
|
|
size_t pagesize = huge_page_size(h);
|
|
|
|
|
|
|
|
if (!__hugetlb_valid_size(pagesize)) {
|
|
|
|
pr_warn("%s: unrecognized huge page size 0x%lx\n",
|
|
|
|
__func__, pagesize);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
2019-03-06 07:43:58 +08:00
|
|
|
#endif
|
|
|
|
|
2013-04-10 20:48:00 +08:00
|
|
|
int pmd_huge(pmd_t pmd)
|
|
|
|
{
|
2015-07-01 20:08:31 +08:00
|
|
|
return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
|
2013-04-10 20:48:00 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
int pud_huge(pud_t pud)
|
|
|
|
{
|
2014-05-15 22:19:22 +08:00
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
2015-07-01 20:08:31 +08:00
|
|
|
return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT);
|
2014-05-15 22:19:22 +08:00
|
|
|
#else
|
|
|
|
return 0;
|
|
|
|
#endif
|
2013-04-10 20:48:00 +08:00
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:42 +08:00
|
|
|
/*
|
|
|
|
* Select all bits except the pfn
|
|
|
|
*/
|
|
|
|
static inline pgprot_t pte_pgprot(pte_t pte)
|
|
|
|
{
|
|
|
|
unsigned long pfn = pte_pfn(pte);
|
|
|
|
|
|
|
|
return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
|
|
|
|
}
|
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
static int find_num_contig(struct mm_struct *mm, unsigned long addr,
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-07 06:39:29 +08:00
|
|
|
pte_t *ptep, size_t *pgsize)
|
2015-12-18 03:31:26 +08:00
|
|
|
{
|
2018-02-15 19:14:56 +08:00
|
|
|
pgd_t *pgdp = pgd_offset(mm, addr);
|
2020-06-05 07:46:23 +08:00
|
|
|
p4d_t *p4dp;
|
2018-02-15 19:14:56 +08:00
|
|
|
pud_t *pudp;
|
|
|
|
pmd_t *pmdp;
|
2015-12-18 03:31:26 +08:00
|
|
|
|
|
|
|
*pgsize = PAGE_SIZE;
|
2020-06-05 07:46:23 +08:00
|
|
|
p4dp = p4d_offset(pgdp, addr);
|
|
|
|
pudp = pud_offset(p4dp, addr);
|
2018-02-15 19:14:56 +08:00
|
|
|
pmdp = pmd_offset(pudp, addr);
|
|
|
|
if ((pte_t *)pmdp == ptep) {
|
2015-12-18 03:31:26 +08:00
|
|
|
*pgsize = PMD_SIZE;
|
|
|
|
return CONT_PMDS;
|
|
|
|
}
|
|
|
|
return CONT_PTES;
|
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:46 +08:00
|
|
|
static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
|
|
|
|
{
|
|
|
|
int contig_ptes = 0;
|
|
|
|
|
|
|
|
*pgsize = size;
|
|
|
|
|
|
|
|
switch (size) {
|
2021-09-20 17:29:31 +08:00
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
2017-08-22 18:42:46 +08:00
|
|
|
case PUD_SIZE:
|
2021-09-20 17:29:31 +08:00
|
|
|
if (pud_sect_supported())
|
|
|
|
contig_ptes = 1;
|
|
|
|
break;
|
2017-08-22 18:42:46 +08:00
|
|
|
#endif
|
|
|
|
case PMD_SIZE:
|
|
|
|
contig_ptes = 1;
|
|
|
|
break;
|
|
|
|
case CONT_PMD_SIZE:
|
|
|
|
*pgsize = PMD_SIZE;
|
|
|
|
contig_ptes = CONT_PMDS;
|
|
|
|
break;
|
|
|
|
case CONT_PTE_SIZE:
|
|
|
|
*pgsize = PAGE_SIZE;
|
|
|
|
contig_ptes = CONT_PTES;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return contig_ptes;
|
|
|
|
}
|
|
|
|
|
2022-05-16 08:55:58 +08:00
|
|
|
pte_t huge_ptep_get(pte_t *ptep)
|
|
|
|
{
|
|
|
|
int ncontig, i;
|
|
|
|
size_t pgsize;
|
|
|
|
pte_t orig_pte = ptep_get(ptep);
|
|
|
|
|
|
|
|
if (!pte_present(orig_pte) || !pte_cont(orig_pte))
|
|
|
|
return orig_pte;
|
|
|
|
|
|
|
|
ncontig = num_contig_ptes(page_size(pte_page(orig_pte)), &pgsize);
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++) {
|
|
|
|
pte_t pte = ptep_get(ptep);
|
|
|
|
|
|
|
|
if (pte_dirty(pte))
|
|
|
|
orig_pte = pte_mkdirty(orig_pte);
|
|
|
|
|
|
|
|
if (pte_young(pte))
|
|
|
|
orig_pte = pte_mkyoung(orig_pte);
|
|
|
|
}
|
|
|
|
return orig_pte;
|
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:44 +08:00
|
|
|
/*
|
|
|
|
* Changing some bits of contiguous entries requires us to follow a
|
|
|
|
* Break-Before-Make approach, breaking the whole contiguous set
|
|
|
|
* before we can change any entries. See ARM DDI 0487A.k_iss10775,
|
|
|
|
* "Misprogramming of the Contiguous bit", page D4-1762.
|
|
|
|
*
|
|
|
|
* This helper performs the break step.
|
|
|
|
*/
|
2022-05-10 12:39:30 +08:00
|
|
|
static pte_t get_clear_contig(struct mm_struct *mm,
|
2017-08-22 18:42:44 +08:00
|
|
|
unsigned long addr,
|
|
|
|
pte_t *ptep,
|
|
|
|
unsigned long pgsize,
|
|
|
|
unsigned long ncontig)
|
|
|
|
{
|
2022-05-16 08:55:57 +08:00
|
|
|
pte_t orig_pte = ptep_get(ptep);
|
2022-05-10 12:39:30 +08:00
|
|
|
unsigned long i;
|
2017-08-22 18:42:44 +08:00
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
|
|
|
|
pte_t pte = ptep_get_and_clear(mm, addr, ptep);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If HW_AFDBM is enabled, then the HW could turn on
|
2018-09-21 23:34:04 +08:00
|
|
|
* the dirty or accessed bit for any page in the set,
|
|
|
|
* so check them all.
|
2017-08-22 18:42:44 +08:00
|
|
|
*/
|
|
|
|
if (pte_dirty(pte))
|
|
|
|
orig_pte = pte_mkdirty(orig_pte);
|
2018-09-21 23:34:04 +08:00
|
|
|
|
|
|
|
if (pte_young(pte))
|
|
|
|
orig_pte = pte_mkyoung(orig_pte);
|
2017-08-22 18:42:44 +08:00
|
|
|
}
|
|
|
|
return orig_pte;
|
|
|
|
}
|
|
|
|
|
2022-06-29 17:53:49 +08:00
|
|
|
static pte_t get_clear_contig_flush(struct mm_struct *mm,
|
|
|
|
unsigned long addr,
|
|
|
|
pte_t *ptep,
|
|
|
|
unsigned long pgsize,
|
|
|
|
unsigned long ncontig)
|
|
|
|
{
|
|
|
|
pte_t orig_pte = get_clear_contig(mm, addr, ptep, pgsize, ncontig);
|
|
|
|
struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
|
|
|
|
|
|
|
|
flush_tlb_range(&vma, addr, addr + (pgsize * ncontig));
|
|
|
|
return orig_pte;
|
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:44 +08:00
|
|
|
/*
|
|
|
|
* Changing some bits of contiguous entries requires us to follow a
|
|
|
|
* Break-Before-Make approach, breaking the whole contiguous set
|
|
|
|
* before we can change any entries. See ARM DDI 0487A.k_iss10775,
|
|
|
|
* "Misprogramming of the Contiguous bit", page D4-1762.
|
|
|
|
*
|
|
|
|
* This helper performs the break step for use cases where the
|
|
|
|
* original pte is not needed.
|
|
|
|
*/
|
|
|
|
static void clear_flush(struct mm_struct *mm,
|
|
|
|
unsigned long addr,
|
|
|
|
pte_t *ptep,
|
|
|
|
unsigned long pgsize,
|
|
|
|
unsigned long ncontig)
|
|
|
|
{
|
mm: do not initialize TLB stack vma's with vma_init()
Commit 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and
data segments") tried to initialize various left-over ad-hoc vma's
"properly", but actually made things worse for the temporary vma's used
for TLB flushing.
vma_init() doesn't actually initialize all of the vma, just a few
fields, so doing something like
- struct vm_area_struct vma = { .vm_mm = tlb->mm, };
+ struct vm_area_struct vma;
+
+ vma_init(&vma, tlb->mm);
was actually very bad: instead of having a nicely initialized vma with
every field but "vm_mm" zeroed, you'd have an entirely uninitialized vma
with only a couple of fields initialized. And they weren't even fields
that the code in question mostly cared about.
The flush_tlb_range() function takes a "struct vma" rather than a
"struct mm_struct", because a few architectures actually care about what
kind of range it is - being able to only do an ITLB flush if it's a
range that doesn't have data accesses enabled, for example. And all the
normal users already have the vma for doing the range invalidation.
But a few people want to call flush_tlb_range() with a range they just
made up, so they also end up using a made-up vma. x86 just has a
special "flush_tlb_mm_range()" function for this, but other
architectures (arm and ia64) do the "use fake vma" thing instead, and
thus got caught up in the vma_init() changes.
At the same time, the TLB flushing code really doesn't care about most
other fields in the vma, so vma_init() is just unnecessary and
pointless.
This fixes things by having an explicit "this is just an initializer for
the TLB flush" initializer macro, which is used by the arm/arm64/ia64
people who mis-use this interface with just a dummy vma.
Fixes: 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and data segments")
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Kirill Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2018-08-02 04:43:38 +08:00
|
|
|
struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
|
2017-08-22 18:42:44 +08:00
|
|
|
unsigned long i, saddr = addr;
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
|
|
|
|
pte_clear(mm, addr, ptep);
|
|
|
|
|
|
|
|
flush_tlb_range(&vma, saddr, addr);
|
|
|
|
}
|
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t *ptep, pte_t pte)
|
|
|
|
{
|
|
|
|
size_t pgsize;
|
|
|
|
int i;
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-07 06:39:29 +08:00
|
|
|
int ncontig;
|
2017-08-22 18:42:43 +08:00
|
|
|
unsigned long pfn, dpfn;
|
2015-12-18 03:31:26 +08:00
|
|
|
pgprot_t hugeprot;
|
|
|
|
|
2017-08-22 18:42:41 +08:00
|
|
|
/*
|
|
|
|
* Code needs to be expanded to handle huge swap and migration
|
|
|
|
* entries. Needed for HUGETLB and MEMORY_FAILURE.
|
|
|
|
*/
|
|
|
|
WARN_ON(!pte_present(pte));
|
|
|
|
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-07 06:39:29 +08:00
|
|
|
if (!pte_cont(pte)) {
|
2015-12-18 03:31:26 +08:00
|
|
|
set_pte_at(mm, addr, ptep, pte);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-07 06:39:29 +08:00
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
2015-12-18 03:31:26 +08:00
|
|
|
pfn = pte_pfn(pte);
|
2017-08-22 18:42:43 +08:00
|
|
|
dpfn = pgsize >> PAGE_SHIFT;
|
2017-08-22 18:42:42 +08:00
|
|
|
hugeprot = pte_pgprot(pte);
|
2017-08-22 18:42:43 +08:00
|
|
|
|
2017-08-22 18:42:44 +08:00
|
|
|
clear_flush(mm, addr, ptep, pgsize, ncontig);
|
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
2015-12-18 03:31:26 +08:00
|
|
|
set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
|
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:47 +08:00
|
|
|
void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t *ptep, pte_t pte, unsigned long sz)
|
|
|
|
{
|
|
|
|
int i, ncontig;
|
|
|
|
size_t pgsize;
|
|
|
|
|
|
|
|
ncontig = num_contig_ptes(sz, &pgsize);
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++)
|
|
|
|
set_pte(ptep, pte);
|
|
|
|
}
|
|
|
|
|
2021-05-05 09:33:00 +08:00
|
|
|
pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
|
2015-12-18 03:31:26 +08:00
|
|
|
unsigned long addr, unsigned long sz)
|
|
|
|
{
|
2018-02-15 19:14:56 +08:00
|
|
|
pgd_t *pgdp;
|
2020-06-05 07:46:23 +08:00
|
|
|
p4d_t *p4dp;
|
2018-02-15 19:14:56 +08:00
|
|
|
pud_t *pudp;
|
|
|
|
pmd_t *pmdp;
|
|
|
|
pte_t *ptep = NULL;
|
|
|
|
|
|
|
|
pgdp = pgd_offset(mm, addr);
|
2020-06-05 07:46:23 +08:00
|
|
|
p4dp = p4d_offset(pgdp, addr);
|
|
|
|
pudp = pud_alloc(mm, p4dp, addr);
|
2018-02-15 19:14:56 +08:00
|
|
|
if (!pudp)
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (sz == PUD_SIZE) {
|
2018-02-15 19:14:56 +08:00
|
|
|
ptep = (pte_t *)pudp;
|
2019-05-21 11:35:03 +08:00
|
|
|
} else if (sz == (CONT_PTE_SIZE)) {
|
2018-02-15 19:14:56 +08:00
|
|
|
pmdp = pmd_alloc(mm, pudp, addr);
|
2020-05-05 20:59:30 +08:00
|
|
|
if (!pmdp)
|
|
|
|
return NULL;
|
2015-12-18 03:31:26 +08:00
|
|
|
|
|
|
|
WARN_ON(addr & (sz - 1));
|
|
|
|
/*
|
|
|
|
* Note that if this code were ever ported to the
|
|
|
|
* 32-bit arm platform then it will cause trouble in
|
|
|
|
* the case where CONFIG_HIGHPTE is set, since there
|
|
|
|
* will be no pte_unmap() to correspond with this
|
|
|
|
* pte_alloc_map().
|
|
|
|
*/
|
2018-02-15 19:14:56 +08:00
|
|
|
ptep = pte_alloc_map(mm, pmdp, addr);
|
2015-12-18 03:31:26 +08:00
|
|
|
} else if (sz == PMD_SIZE) {
|
2021-05-05 09:33:04 +08:00
|
|
|
if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp)))
|
2021-05-05 09:33:00 +08:00
|
|
|
ptep = huge_pmd_share(mm, vma, addr, pudp);
|
2015-12-18 03:31:26 +08:00
|
|
|
else
|
2018-02-15 19:14:56 +08:00
|
|
|
ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
|
2019-05-21 11:35:03 +08:00
|
|
|
} else if (sz == (CONT_PMD_SIZE)) {
|
2018-02-15 19:14:56 +08:00
|
|
|
pmdp = pmd_alloc(mm, pudp, addr);
|
2015-12-18 03:31:26 +08:00
|
|
|
WARN_ON(addr & (sz - 1));
|
2018-02-15 19:14:56 +08:00
|
|
|
return (pte_t *)pmdp;
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
return ptep;
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
2017-07-07 06:39:42 +08:00
|
|
|
pte_t *huge_pte_offset(struct mm_struct *mm,
|
|
|
|
unsigned long addr, unsigned long sz)
|
2015-12-18 03:31:26 +08:00
|
|
|
{
|
2018-02-15 19:14:56 +08:00
|
|
|
pgd_t *pgdp;
|
2020-06-05 07:46:23 +08:00
|
|
|
p4d_t *p4dp;
|
2018-02-15 19:14:56 +08:00
|
|
|
pud_t *pudp, pud;
|
|
|
|
pmd_t *pmdp, pmd;
|
2015-12-18 03:31:26 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
pgdp = pgd_offset(mm, addr);
|
|
|
|
if (!pgd_present(READ_ONCE(*pgdp)))
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
2017-06-09 01:25:26 +08:00
|
|
|
|
2020-06-05 07:46:23 +08:00
|
|
|
p4dp = p4d_offset(pgdp, addr);
|
|
|
|
if (!p4d_present(READ_ONCE(*p4dp)))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
pudp = pud_offset(p4dp, addr);
|
2018-02-15 19:14:56 +08:00
|
|
|
pud = READ_ONCE(*pudp);
|
|
|
|
if (sz != PUD_SIZE && pud_none(pud))
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
2017-08-22 18:42:45 +08:00
|
|
|
/* hugepage or swap? */
|
2018-02-15 19:14:56 +08:00
|
|
|
if (pud_huge(pud) || !pud_present(pud))
|
|
|
|
return (pte_t *)pudp;
|
2017-06-09 01:25:26 +08:00
|
|
|
/* table; check the next level */
|
|
|
|
|
2017-08-22 18:42:45 +08:00
|
|
|
if (sz == CONT_PMD_SIZE)
|
|
|
|
addr &= CONT_PMD_MASK;
|
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
pmdp = pmd_offset(pudp, addr);
|
|
|
|
pmd = READ_ONCE(*pmdp);
|
2017-08-22 18:42:45 +08:00
|
|
|
if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
|
2018-02-15 19:14:56 +08:00
|
|
|
pmd_none(pmd))
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
2018-02-15 19:14:56 +08:00
|
|
|
if (pmd_huge(pmd) || !pmd_present(pmd))
|
|
|
|
return (pte_t *)pmdp;
|
2017-06-09 01:25:26 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
if (sz == CONT_PTE_SIZE)
|
|
|
|
return pte_offset_kernel(pmdp, (addr & CONT_PTE_MASK));
|
2017-08-22 18:42:45 +08:00
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
mm/hugetlb: change parameters of arch_make_huge_pte()
Patch series "Subject: [PATCH v2 0/5] Implement huge VMAP and VMALLOC on powerpc 8xx", v2.
This series implements huge VMAP and VMALLOC on powerpc 8xx.
Powerpc 8xx has 4 page sizes:
- 4k
- 16k
- 512k
- 8M
At the time being, vmalloc and vmap only support huge pages which are
leaf at PMD level.
Here the PMD level is 4M, it doesn't correspond to any supported
page size.
For now, implement use of 16k and 512k pages which is done
at PTE level.
Support of 8M pages will be implemented later, it requires use of
hugepd tables.
To allow this, the architecture provides two functions:
- arch_vmap_pte_range_map_size() which tells vmap_pte_range() what
page size to use. A stub returning PAGE_SIZE is provided when the
architecture doesn't provide this function.
- arch_vmap_pte_supported_shift() which tells __vmalloc_node_range()
what page shift to use for a given area size. A stub returning
PAGE_SHIFT is provided when the architecture doesn't provide this
function.
This patch (of 5):
At the time being, arch_make_huge_pte() has the following prototype:
pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
struct page *page, int writable);
vma is used to get the pages shift or size.
vma is also used on Sparc to get vm_flags.
page is not used.
writable is not used.
In order to use this function without a vma, replace vma by shift and
flags. Also remove the used parameters.
Link: https://lkml.kernel.org/r/cover.1620795204.git.christophe.leroy@csgroup.eu
Link: https://lkml.kernel.org/r/f4633ac6a7da2f22f31a04a89e0a7026bb78b15b.1620795204.git.christophe.leroy@csgroup.eu
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Acked-by: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Uladzislau Rezki <uladzislau.rezki@sony.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2021-07-01 09:48:00 +08:00
|
|
|
pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
|
2015-12-18 03:31:26 +08:00
|
|
|
{
|
mm/hugetlb: change parameters of arch_make_huge_pte()
Patch series "Subject: [PATCH v2 0/5] Implement huge VMAP and VMALLOC on powerpc 8xx", v2.
This series implements huge VMAP and VMALLOC on powerpc 8xx.
Powerpc 8xx has 4 page sizes:
- 4k
- 16k
- 512k
- 8M
At the time being, vmalloc and vmap only support huge pages which are
leaf at PMD level.
Here the PMD level is 4M, it doesn't correspond to any supported
page size.
For now, implement use of 16k and 512k pages which is done
at PTE level.
Support of 8M pages will be implemented later, it requires use of
hugepd tables.
To allow this, the architecture provides two functions:
- arch_vmap_pte_range_map_size() which tells vmap_pte_range() what
page size to use. A stub returning PAGE_SIZE is provided when the
architecture doesn't provide this function.
- arch_vmap_pte_supported_shift() which tells __vmalloc_node_range()
what page shift to use for a given area size. A stub returning
PAGE_SHIFT is provided when the architecture doesn't provide this
function.
This patch (of 5):
At the time being, arch_make_huge_pte() has the following prototype:
pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
struct page *page, int writable);
vma is used to get the pages shift or size.
vma is also used on Sparc to get vm_flags.
page is not used.
writable is not used.
In order to use this function without a vma, replace vma by shift and
flags. Also remove the used parameters.
Link: https://lkml.kernel.org/r/cover.1620795204.git.christophe.leroy@csgroup.eu
Link: https://lkml.kernel.org/r/f4633ac6a7da2f22f31a04a89e0a7026bb78b15b.1620795204.git.christophe.leroy@csgroup.eu
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Acked-by: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Uladzislau Rezki <uladzislau.rezki@sony.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2021-07-01 09:48:00 +08:00
|
|
|
size_t pagesize = 1UL << shift;
|
2015-12-18 03:31:26 +08:00
|
|
|
|
2022-03-23 05:41:47 +08:00
|
|
|
entry = pte_mkhuge(entry);
|
2015-12-18 03:31:26 +08:00
|
|
|
if (pagesize == CONT_PTE_SIZE) {
|
|
|
|
entry = pte_mkcont(entry);
|
|
|
|
} else if (pagesize == CONT_PMD_SIZE) {
|
|
|
|
entry = pmd_pte(pmd_mkcont(pte_pmd(entry)));
|
|
|
|
} else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) {
|
|
|
|
pr_warn("%s: unrecognized huge page size 0x%lx\n",
|
|
|
|
__func__, pagesize);
|
|
|
|
}
|
|
|
|
return entry;
|
|
|
|
}
|
|
|
|
|
2017-08-22 18:42:46 +08:00
|
|
|
void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t *ptep, unsigned long sz)
|
|
|
|
{
|
|
|
|
int i, ncontig;
|
|
|
|
size_t pgsize;
|
|
|
|
|
|
|
|
ncontig = num_contig_ptes(sz, &pgsize);
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
|
|
|
|
pte_clear(mm, addr, ptep);
|
|
|
|
}
|
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
|
|
|
{
|
2017-08-22 18:42:44 +08:00
|
|
|
int ncontig;
|
2017-08-22 18:42:43 +08:00
|
|
|
size_t pgsize;
|
2022-05-16 08:55:57 +08:00
|
|
|
pte_t orig_pte = ptep_get(ptep);
|
2017-08-22 18:42:43 +08:00
|
|
|
|
|
|
|
if (!pte_cont(orig_pte))
|
2015-12-18 03:31:26 +08:00
|
|
|
return ptep_get_and_clear(mm, addr, ptep);
|
2017-08-22 18:42:43 +08:00
|
|
|
|
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
|
|
|
|
2022-05-10 12:39:30 +08:00
|
|
|
return get_clear_contig(mm, addr, ptep, pgsize, ncontig);
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
2018-09-21 23:34:05 +08:00
|
|
|
/*
|
|
|
|
* huge_ptep_set_access_flags will update access flags (dirty, accesssed)
|
|
|
|
* and write permission.
|
|
|
|
*
|
|
|
|
* For a contiguous huge pte range we need to check whether or not write
|
|
|
|
* permission has to change only on the first pte in the set. Then for
|
|
|
|
* all the contiguous ptes we need to check whether or not there is a
|
|
|
|
* discrepancy between dirty or young.
|
|
|
|
*/
|
|
|
|
static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2022-05-16 08:55:57 +08:00
|
|
|
if (pte_write(pte) != pte_write(ptep_get(ptep)))
|
2018-09-21 23:34:05 +08:00
|
|
|
return 1;
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++) {
|
2022-05-16 08:55:57 +08:00
|
|
|
pte_t orig_pte = ptep_get(ptep + i);
|
2018-09-21 23:34:05 +08:00
|
|
|
|
|
|
|
if (pte_dirty(pte) != pte_dirty(orig_pte))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
if (pte_young(pte) != pte_young(orig_pte))
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-12-18 03:31:26 +08:00
|
|
|
int huge_ptep_set_access_flags(struct vm_area_struct *vma,
|
|
|
|
unsigned long addr, pte_t *ptep,
|
|
|
|
pte_t pte, int dirty)
|
|
|
|
{
|
2018-09-21 23:34:05 +08:00
|
|
|
int ncontig, i;
|
2017-08-22 18:42:43 +08:00
|
|
|
size_t pgsize = 0;
|
|
|
|
unsigned long pfn = pte_pfn(pte), dpfn;
|
2022-06-29 17:53:49 +08:00
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
2017-08-22 18:42:43 +08:00
|
|
|
pgprot_t hugeprot;
|
2017-08-22 18:42:44 +08:00
|
|
|
pte_t orig_pte;
|
2017-08-22 18:42:43 +08:00
|
|
|
|
|
|
|
if (!pte_cont(pte))
|
2015-12-18 03:31:26 +08:00
|
|
|
return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
|
2017-08-22 18:42:43 +08:00
|
|
|
|
2022-06-29 17:53:49 +08:00
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
2017-08-22 18:42:43 +08:00
|
|
|
dpfn = pgsize >> PAGE_SHIFT;
|
|
|
|
|
2018-09-21 23:34:05 +08:00
|
|
|
if (!__cont_access_flags_changed(ptep, pte, ncontig))
|
|
|
|
return 0;
|
|
|
|
|
2022-06-29 17:53:49 +08:00
|
|
|
orig_pte = get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
|
2017-08-22 18:42:44 +08:00
|
|
|
|
2018-09-21 23:34:04 +08:00
|
|
|
/* Make sure we don't lose the dirty or young state */
|
2017-08-22 18:42:44 +08:00
|
|
|
if (pte_dirty(orig_pte))
|
|
|
|
pte = pte_mkdirty(pte);
|
|
|
|
|
2018-09-21 23:34:04 +08:00
|
|
|
if (pte_young(orig_pte))
|
|
|
|
pte = pte_mkyoung(pte);
|
|
|
|
|
2017-08-22 18:42:44 +08:00
|
|
|
hugeprot = pte_pgprot(pte);
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
2022-06-29 17:53:49 +08:00
|
|
|
set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
|
2017-08-22 18:42:43 +08:00
|
|
|
|
2018-09-21 23:34:05 +08:00
|
|
|
return 1;
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void huge_ptep_set_wrprotect(struct mm_struct *mm,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
|
|
|
{
|
2017-08-22 18:42:44 +08:00
|
|
|
unsigned long pfn, dpfn;
|
|
|
|
pgprot_t hugeprot;
|
2017-08-22 18:42:43 +08:00
|
|
|
int ncontig, i;
|
|
|
|
size_t pgsize;
|
2017-08-22 18:42:44 +08:00
|
|
|
pte_t pte;
|
2015-12-18 03:31:26 +08:00
|
|
|
|
2018-02-15 19:14:56 +08:00
|
|
|
if (!pte_cont(READ_ONCE(*ptep))) {
|
2015-12-18 03:31:26 +08:00
|
|
|
ptep_set_wrprotect(mm, addr, ptep);
|
2017-08-22 18:42:43 +08:00
|
|
|
return;
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
2017-08-22 18:42:43 +08:00
|
|
|
|
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
2017-08-22 18:42:44 +08:00
|
|
|
dpfn = pgsize >> PAGE_SHIFT;
|
|
|
|
|
2022-06-29 17:53:49 +08:00
|
|
|
pte = get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
|
2017-08-22 18:42:44 +08:00
|
|
|
pte = pte_wrprotect(pte);
|
|
|
|
|
|
|
|
hugeprot = pte_pgprot(pte);
|
|
|
|
pfn = pte_pfn(pte);
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
|
|
|
set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
2022-05-14 07:48:55 +08:00
|
|
|
pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
2015-12-18 03:31:26 +08:00
|
|
|
{
|
2022-06-29 17:53:49 +08:00
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
2017-08-22 18:42:43 +08:00
|
|
|
size_t pgsize;
|
2017-08-22 18:42:44 +08:00
|
|
|
int ncontig;
|
2017-08-22 18:42:43 +08:00
|
|
|
|
2022-05-14 07:48:55 +08:00
|
|
|
if (!pte_cont(READ_ONCE(*ptep)))
|
|
|
|
return ptep_clear_flush(vma, addr, ptep);
|
2017-08-22 18:42:43 +08:00
|
|
|
|
2022-06-29 17:53:49 +08:00
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
|
|
|
return get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
|
2015-12-18 03:31:26 +08:00
|
|
|
}
|
|
|
|
|
2018-10-23 09:06:57 +08:00
|
|
|
static int __init hugetlbpage_init(void)
|
|
|
|
{
|
2021-09-20 17:29:31 +08:00
|
|
|
if (pud_sect_supported())
|
|
|
|
hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
|
|
|
|
|
arm64/mm: Redefine CONT_{PTE, PMD}_SHIFT
Currently, the value of CONT_{PTE, PMD}_SHIFT is off from standard
{PAGE, PMD}_SHIFT. In turn, we have to consider adding {PAGE, PMD}_SHIFT
when using CONT_{PTE, PMD}_SHIFT in the function hugetlbpage_init().
It's a bit confusing.
This redefines CONT_{PTE, PMD}_SHIFT with {PAGE, PMD}_SHIFT included
so that the later values needn't be added when using the former ones
in function hugetlbpage_init(). Note that the values of CONT_{PTES, PMDS}
are unchanged.
Suggested-by: Will Deacon <will@kernel.org>
Signed-off-by: Gavin Shan <gshan@redhat.com>
Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lkml.org/lkml/2020/5/6/190
Link: https://lore.kernel.org/r/20200630062428.194235-1-gshan@redhat.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2020-06-30 14:24:28 +08:00
|
|
|
hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT);
|
2020-06-04 07:00:42 +08:00
|
|
|
hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
|
arm64/mm: Redefine CONT_{PTE, PMD}_SHIFT
Currently, the value of CONT_{PTE, PMD}_SHIFT is off from standard
{PAGE, PMD}_SHIFT. In turn, we have to consider adding {PAGE, PMD}_SHIFT
when using CONT_{PTE, PMD}_SHIFT in the function hugetlbpage_init().
It's a bit confusing.
This redefines CONT_{PTE, PMD}_SHIFT with {PAGE, PMD}_SHIFT included
so that the later values needn't be added when using the former ones
in function hugetlbpage_init(). Note that the values of CONT_{PTES, PMDS}
are unchanged.
Suggested-by: Will Deacon <will@kernel.org>
Signed-off-by: Gavin Shan <gshan@redhat.com>
Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lkml.org/lkml/2020/5/6/190
Link: https://lore.kernel.org/r/20200630062428.194235-1-gshan@redhat.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2020-06-30 14:24:28 +08:00
|
|
|
hugetlb_add_hstate(CONT_PTE_SHIFT - PAGE_SHIFT);
|
2018-10-23 09:06:57 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
arch_initcall(hugetlbpage_init);
|
|
|
|
|
2020-06-04 07:00:34 +08:00
|
|
|
bool __init arch_hugetlb_valid_size(unsigned long size)
|
2013-04-10 20:48:00 +08:00
|
|
|
{
|
2022-02-17 12:52:37 +08:00
|
|
|
return __hugetlb_valid_size(size);
|
2020-06-04 07:00:34 +08:00
|
|
|
}
|