mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-11-11 12:28:41 +08:00
arm64/mm: new ptep layer to manage contig bit
Create a new layer for the in-table PTE manipulation APIs. For now, The existing API is prefixed with double underscore to become the arch-private API and the public API is just a simple wrapper that calls the private API. The public API implementation will subsequently be used to transparently manipulate the contiguous bit where appropriate. But since there are already some contig-aware users (e.g. hugetlb, kernel mapper), we must first ensure those users use the private API directly so that the future contig-bit manipulations in the public API do not interfere with those existing uses. The following APIs are treated this way: - ptep_get - set_pte - set_ptes - pte_clear - ptep_get_and_clear - ptep_test_and_clear_young - ptep_clear_flush_young - ptep_set_wrprotect - ptep_set_access_flags Link: https://lkml.kernel.org/r/20240215103205.2607016-11-ryan.roberts@arm.com Signed-off-by: Ryan Roberts <ryan.roberts@arm.com> Tested-by: John Hubbard <jhubbard@nvidia.com> Acked-by: Mark Rutland <mark.rutland@arm.com> Acked-by: Catalin Marinas <catalin.marinas@arm.com> Cc: Alistair Popple <apopple@nvidia.com> Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com> Cc: Ard Biesheuvel <ardb@kernel.org> Cc: Barry Song <21cnbao@gmail.com> Cc: Borislav Petkov (AMD) <bp@alien8.de> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: David Hildenbrand <david@redhat.com> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: Ingo Molnar <mingo@redhat.com> Cc: James Morse <james.morse@arm.com> Cc: Kefeng Wang <wangkefeng.wang@huawei.com> Cc: Marc Zyngier <maz@kernel.org> Cc: Matthew Wilcox (Oracle) <willy@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Will Deacon <will@kernel.org> Cc: Yang Shi <shy828301@gmail.com> Cc: Zi Yan <ziy@nvidia.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Ryan Roberts
Andrew Morton
parent
cbb0294fdd
commit
5a00bfd6a5
@ -93,7 +93,8 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
|
||||
__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
|
||||
|
||||
#define pte_none(pte) (!pte_val(pte))
|
||||
#define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0))
|
||||
#define __pte_clear(mm, addr, ptep) \
|
||||
__set_pte(ptep, __pte(0))
|
||||
#define pte_page(pte) (pfn_to_page(pte_pfn(pte)))
|
||||
|
||||
/*
|
||||
@ -137,7 +138,7 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
|
||||
* so that we don't erroneously return false for pages that have been
|
||||
* remapped as PROT_NONE but are yet to be flushed from the TLB.
|
||||
* Note that we can't make any assumptions based on the state of the access
|
||||
* flag, since ptep_clear_flush_young() elides a DSB when invalidating the
|
||||
* flag, since __ptep_clear_flush_young() elides a DSB when invalidating the
|
||||
* TLB.
|
||||
*/
|
||||
#define pte_accessible(mm, pte) \
|
||||
@ -261,7 +262,7 @@ static inline pte_t pte_mkdevmap(pte_t pte)
|
||||
return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL));
|
||||
}
|
||||
|
||||
static inline void set_pte(pte_t *ptep, pte_t pte)
|
||||
static inline void __set_pte(pte_t *ptep, pte_t pte)
|
||||
{
|
||||
WRITE_ONCE(*ptep, pte);
|
||||
|
||||
@ -275,8 +276,7 @@ static inline void set_pte(pte_t *ptep, pte_t pte)
|
||||
}
|
||||
}
|
||||
|
||||
#define ptep_get ptep_get
|
||||
static inline pte_t ptep_get(pte_t *ptep)
|
||||
static inline pte_t __ptep_get(pte_t *ptep)
|
||||
{
|
||||
return READ_ONCE(*ptep);
|
||||
}
|
||||
@ -308,7 +308,7 @@ static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep,
|
||||
if (!IS_ENABLED(CONFIG_DEBUG_VM))
|
||||
return;
|
||||
|
||||
old_pte = ptep_get(ptep);
|
||||
old_pte = __ptep_get(ptep);
|
||||
|
||||
if (!pte_valid(old_pte) || !pte_valid(pte))
|
||||
return;
|
||||
@ -317,7 +317,7 @@ static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep,
|
||||
|
||||
/*
|
||||
* Check for potential race with hardware updates of the pte
|
||||
* (ptep_set_access_flags safely changes valid ptes without going
|
||||
* (__ptep_set_access_flags safely changes valid ptes without going
|
||||
* through an invalid entry).
|
||||
*/
|
||||
VM_WARN_ONCE(!pte_young(pte),
|
||||
@ -363,23 +363,22 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr)
|
||||
return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));
|
||||
}
|
||||
|
||||
static inline void set_ptes(struct mm_struct *mm,
|
||||
unsigned long __always_unused addr,
|
||||
pte_t *ptep, pte_t pte, unsigned int nr)
|
||||
static inline void __set_ptes(struct mm_struct *mm,
|
||||
unsigned long __always_unused addr,
|
||||
pte_t *ptep, pte_t pte, unsigned int nr)
|
||||
{
|
||||
page_table_check_ptes_set(mm, ptep, pte, nr);
|
||||
__sync_cache_and_tags(pte, nr);
|
||||
|
||||
for (;;) {
|
||||
__check_safe_pte_update(mm, ptep, pte);
|
||||
set_pte(ptep, pte);
|
||||
__set_pte(ptep, pte);
|
||||
if (--nr == 0)
|
||||
break;
|
||||
ptep++;
|
||||
pte = pte_advance_pfn(pte, 1);
|
||||
}
|
||||
}
|
||||
#define set_ptes set_ptes
|
||||
|
||||
/*
|
||||
* Huge pte definitions.
|
||||
@ -546,7 +545,7 @@ static inline void __set_pte_at(struct mm_struct *mm,
|
||||
{
|
||||
__sync_cache_and_tags(pte, nr);
|
||||
__check_safe_pte_update(mm, ptep, pte);
|
||||
set_pte(ptep, pte);
|
||||
__set_pte(ptep, pte);
|
||||
}
|
||||
|
||||
static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
||||
@ -860,8 +859,7 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
||||
return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
|
||||
}
|
||||
|
||||
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
|
||||
extern int ptep_set_access_flags(struct vm_area_struct *vma,
|
||||
extern int __ptep_set_access_flags(struct vm_area_struct *vma,
|
||||
unsigned long address, pte_t *ptep,
|
||||
pte_t entry, int dirty);
|
||||
|
||||
@ -871,7 +869,8 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp,
|
||||
pmd_t entry, int dirty)
|
||||
{
|
||||
return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty);
|
||||
return __ptep_set_access_flags(vma, address, (pte_t *)pmdp,
|
||||
pmd_pte(entry), dirty);
|
||||
}
|
||||
|
||||
static inline int pud_devmap(pud_t pud)
|
||||
@ -905,12 +904,13 @@ static inline bool pud_user_accessible_page(pud_t pud)
|
||||
/*
|
||||
* Atomic pte/pmd modifications.
|
||||
*/
|
||||
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
|
||||
static inline int __ptep_test_and_clear_young(pte_t *ptep)
|
||||
static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma,
|
||||
unsigned long address,
|
||||
pte_t *ptep)
|
||||
{
|
||||
pte_t old_pte, pte;
|
||||
|
||||
pte = ptep_get(ptep);
|
||||
pte = __ptep_get(ptep);
|
||||
do {
|
||||
old_pte = pte;
|
||||
pte = pte_mkold(pte);
|
||||
@ -921,18 +921,10 @@ static inline int __ptep_test_and_clear_young(pte_t *ptep)
|
||||
return pte_young(pte);
|
||||
}
|
||||
|
||||
static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
|
||||
unsigned long address,
|
||||
pte_t *ptep)
|
||||
{
|
||||
return __ptep_test_and_clear_young(ptep);
|
||||
}
|
||||
|
||||
#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
|
||||
static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
|
||||
static inline int __ptep_clear_flush_young(struct vm_area_struct *vma,
|
||||
unsigned long address, pte_t *ptep)
|
||||
{
|
||||
int young = ptep_test_and_clear_young(vma, address, ptep);
|
||||
int young = __ptep_test_and_clear_young(vma, address, ptep);
|
||||
|
||||
if (young) {
|
||||
/*
|
||||
@ -955,12 +947,11 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
|
||||
unsigned long address,
|
||||
pmd_t *pmdp)
|
||||
{
|
||||
return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
|
||||
return __ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
|
||||
}
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
|
||||
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
|
||||
static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
|
||||
static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long address, pte_t *ptep)
|
||||
{
|
||||
pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));
|
||||
@ -984,15 +975,15 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
|
||||
/*
|
||||
* ptep_set_wrprotect - mark read-only while trasferring potential hardware
|
||||
* __ptep_set_wrprotect - mark read-only while trasferring potential hardware
|
||||
* dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
|
||||
*/
|
||||
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
|
||||
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
|
||||
static inline void __ptep_set_wrprotect(struct mm_struct *mm,
|
||||
unsigned long address, pte_t *ptep)
|
||||
{
|
||||
pte_t old_pte, pte;
|
||||
|
||||
pte = ptep_get(ptep);
|
||||
pte = __ptep_get(ptep);
|
||||
do {
|
||||
old_pte = pte;
|
||||
pte = pte_wrprotect(pte);
|
||||
@ -1006,7 +997,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addres
|
||||
static inline void pmdp_set_wrprotect(struct mm_struct *mm,
|
||||
unsigned long address, pmd_t *pmdp)
|
||||
{
|
||||
ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
|
||||
__ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
|
||||
}
|
||||
|
||||
#define pmdp_establish pmdp_establish
|
||||
@ -1084,7 +1075,7 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)
|
||||
#endif /* CONFIG_ARM64_MTE */
|
||||
|
||||
/*
|
||||
* On AArch64, the cache coherency is handled via the set_ptes() function.
|
||||
* On AArch64, the cache coherency is handled via the __set_ptes() function.
|
||||
*/
|
||||
static inline void update_mmu_cache_range(struct vm_fault *vmf,
|
||||
struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
|
||||
@ -1136,6 +1127,22 @@ extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
|
||||
extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
|
||||
unsigned long addr, pte_t *ptep,
|
||||
pte_t old_pte, pte_t new_pte);
|
||||
|
||||
#define ptep_get __ptep_get
|
||||
#define set_pte __set_pte
|
||||
#define set_ptes __set_ptes
|
||||
#define pte_clear __pte_clear
|
||||
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
|
||||
#define ptep_get_and_clear __ptep_get_and_clear
|
||||
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
|
||||
#define ptep_test_and_clear_young __ptep_test_and_clear_young
|
||||
#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
|
||||
#define ptep_clear_flush_young __ptep_clear_flush_young
|
||||
#define __HAVE_ARCH_PTEP_SET_WRPROTECT
|
||||
#define ptep_set_wrprotect __ptep_set_wrprotect
|
||||
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
|
||||
#define ptep_set_access_flags __ptep_set_access_flags
|
||||
|
||||
#endif /* !__ASSEMBLY__ */
|
||||
|
||||
#endif /* __ASM_PGTABLE_H */
|
||||
|
||||
@ -103,7 +103,7 @@ static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data)
|
||||
{
|
||||
struct set_perm_data *spd = data;
|
||||
const efi_memory_desc_t *md = spd->md;
|
||||
pte_t pte = ptep_get(ptep);
|
||||
pte_t pte = __ptep_get(ptep);
|
||||
|
||||
if (md->attribute & EFI_MEMORY_RO)
|
||||
pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
|
||||
@ -111,7 +111,7 @@ static int __init set_permissions(pte_t *ptep, unsigned long addr, void *data)
|
||||
pte = set_pte_bit(pte, __pgprot(PTE_PXN));
|
||||
else if (system_supports_bti_kernel() && spd->has_bti)
|
||||
pte = set_pte_bit(pte, __pgprot(PTE_GP));
|
||||
set_pte(ptep, pte);
|
||||
__set_pte(ptep, pte);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -67,7 +67,7 @@ int memcmp_pages(struct page *page1, struct page *page2)
|
||||
/*
|
||||
* If the page content is identical but at least one of the pages is
|
||||
* tagged, return non-zero to avoid KSM merging. If only one of the
|
||||
* pages is tagged, set_ptes() may zero or change the tags of the
|
||||
* pages is tagged, __set_ptes() may zero or change the tags of the
|
||||
* other page via mte_sync_tags().
|
||||
*/
|
||||
if (page_mte_tagged(page1) || page_mte_tagged(page2))
|
||||
|
||||
@ -1072,7 +1072,7 @@ int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm,
|
||||
} else {
|
||||
/*
|
||||
* Only locking to serialise with a concurrent
|
||||
* set_ptes() in the VMM but still overriding the
|
||||
* __set_ptes() in the VMM but still overriding the
|
||||
* tags, hence ignoring the return value.
|
||||
*/
|
||||
try_page_mte_tagging(page);
|
||||
|
||||
@ -191,7 +191,7 @@ static void show_pte(unsigned long addr)
|
||||
if (!ptep)
|
||||
break;
|
||||
|
||||
pte = ptep_get(ptep);
|
||||
pte = __ptep_get(ptep);
|
||||
pr_cont(", pte=%016llx", pte_val(pte));
|
||||
pte_unmap(ptep);
|
||||
} while(0);
|
||||
@ -205,16 +205,16 @@ static void show_pte(unsigned long addr)
|
||||
*
|
||||
* It needs to cope with hardware update of the accessed/dirty state by other
|
||||
* agents in the system and can safely skip the __sync_icache_dcache() call as,
|
||||
* like set_ptes(), the PTE is never changed from no-exec to exec here.
|
||||
* like __set_ptes(), the PTE is never changed from no-exec to exec here.
|
||||
*
|
||||
* Returns whether or not the PTE actually changed.
|
||||
*/
|
||||
int ptep_set_access_flags(struct vm_area_struct *vma,
|
||||
unsigned long address, pte_t *ptep,
|
||||
pte_t entry, int dirty)
|
||||
int __ptep_set_access_flags(struct vm_area_struct *vma,
|
||||
unsigned long address, pte_t *ptep,
|
||||
pte_t entry, int dirty)
|
||||
{
|
||||
pteval_t old_pteval, pteval;
|
||||
pte_t pte = ptep_get(ptep);
|
||||
pte_t pte = __ptep_get(ptep);
|
||||
|
||||
if (pte_same(pte, entry))
|
||||
return 0;
|
||||
|
||||
@ -121,9 +121,9 @@ void __set_fixmap(enum fixed_addresses idx,
|
||||
ptep = fixmap_pte(addr);
|
||||
|
||||
if (pgprot_val(flags)) {
|
||||
set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
|
||||
__set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
|
||||
} else {
|
||||
pte_clear(&init_mm, addr, ptep);
|
||||
__pte_clear(&init_mm, addr, ptep);
|
||||
flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
@ -145,14 +145,14 @@ pte_t huge_ptep_get(pte_t *ptep)
|
||||
{
|
||||
int ncontig, i;
|
||||
size_t pgsize;
|
||||
pte_t orig_pte = ptep_get(ptep);
|
||||
pte_t orig_pte = __ptep_get(ptep);
|
||||
|
||||
if (!pte_present(orig_pte) || !pte_cont(orig_pte))
|
||||
return orig_pte;
|
||||
|
||||
ncontig = num_contig_ptes(page_size(pte_page(orig_pte)), &pgsize);
|
||||
for (i = 0; i < ncontig; i++, ptep++) {
|
||||
pte_t pte = ptep_get(ptep);
|
||||
pte_t pte = __ptep_get(ptep);
|
||||
|
||||
if (pte_dirty(pte))
|
||||
orig_pte = pte_mkdirty(orig_pte);
|
||||
@ -177,11 +177,11 @@ static pte_t get_clear_contig(struct mm_struct *mm,
|
||||
unsigned long pgsize,
|
||||
unsigned long ncontig)
|
||||
{
|
||||
pte_t orig_pte = ptep_get(ptep);
|
||||
pte_t orig_pte = __ptep_get(ptep);
|
||||
unsigned long i;
|
||||
|
||||
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
|
||||
pte_t pte = ptep_get_and_clear(mm, addr, ptep);
|
||||
pte_t pte = __ptep_get_and_clear(mm, addr, ptep);
|
||||
|
||||
/*
|
||||
* If HW_AFDBM is enabled, then the HW could turn on
|
||||
@ -229,7 +229,7 @@ static void clear_flush(struct mm_struct *mm,
|
||||
unsigned long i, saddr = addr;
|
||||
|
||||
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
|
||||
ptep_get_and_clear(mm, addr, ptep);
|
||||
__ptep_get_and_clear(mm, addr, ptep);
|
||||
|
||||
flush_tlb_range(&vma, saddr, addr);
|
||||
}
|
||||
@ -247,12 +247,12 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
|
||||
|
||||
if (!pte_present(pte)) {
|
||||
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize)
|
||||
set_ptes(mm, addr, ptep, pte, 1);
|
||||
__set_ptes(mm, addr, ptep, pte, 1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!pte_cont(pte)) {
|
||||
set_ptes(mm, addr, ptep, pte, 1);
|
||||
__set_ptes(mm, addr, ptep, pte, 1);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -263,7 +263,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
|
||||
clear_flush(mm, addr, ptep, pgsize, ncontig);
|
||||
|
||||
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
||||
set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
|
||||
__set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
|
||||
}
|
||||
|
||||
pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||
@ -393,7 +393,7 @@ void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
|
||||
ncontig = num_contig_ptes(sz, &pgsize);
|
||||
|
||||
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
|
||||
pte_clear(mm, addr, ptep);
|
||||
__pte_clear(mm, addr, ptep);
|
||||
}
|
||||
|
||||
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
|
||||
@ -401,10 +401,10 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
|
||||
{
|
||||
int ncontig;
|
||||
size_t pgsize;
|
||||
pte_t orig_pte = ptep_get(ptep);
|
||||
pte_t orig_pte = __ptep_get(ptep);
|
||||
|
||||
if (!pte_cont(orig_pte))
|
||||
return ptep_get_and_clear(mm, addr, ptep);
|
||||
return __ptep_get_and_clear(mm, addr, ptep);
|
||||
|
||||
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
||||
|
||||
@ -424,11 +424,11 @@ static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (pte_write(pte) != pte_write(ptep_get(ptep)))
|
||||
if (pte_write(pte) != pte_write(__ptep_get(ptep)))
|
||||
return 1;
|
||||
|
||||
for (i = 0; i < ncontig; i++) {
|
||||
pte_t orig_pte = ptep_get(ptep + i);
|
||||
pte_t orig_pte = __ptep_get(ptep + i);
|
||||
|
||||
if (pte_dirty(pte) != pte_dirty(orig_pte))
|
||||
return 1;
|
||||
@ -452,7 +452,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,
|
||||
pte_t orig_pte;
|
||||
|
||||
if (!pte_cont(pte))
|
||||
return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
|
||||
return __ptep_set_access_flags(vma, addr, ptep, pte, dirty);
|
||||
|
||||
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
||||
dpfn = pgsize >> PAGE_SHIFT;
|
||||
@ -471,7 +471,7 @@ int huge_ptep_set_access_flags(struct vm_area_struct *vma,
|
||||
|
||||
hugeprot = pte_pgprot(pte);
|
||||
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
||||
set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
|
||||
__set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
|
||||
|
||||
return 1;
|
||||
}
|
||||
@ -485,8 +485,8 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm,
|
||||
size_t pgsize;
|
||||
pte_t pte;
|
||||
|
||||
if (!pte_cont(ptep_get(ptep))) {
|
||||
ptep_set_wrprotect(mm, addr, ptep);
|
||||
if (!pte_cont(__ptep_get(ptep))) {
|
||||
__ptep_set_wrprotect(mm, addr, ptep);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -500,7 +500,7 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm,
|
||||
pfn = pte_pfn(pte);
|
||||
|
||||
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
||||
set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
|
||||
__set_ptes(mm, addr, ptep, pfn_pte(pfn, hugeprot), 1);
|
||||
}
|
||||
|
||||
pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
|
||||
@ -510,7 +510,7 @@ pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
|
||||
size_t pgsize;
|
||||
int ncontig;
|
||||
|
||||
if (!pte_cont(ptep_get(ptep)))
|
||||
if (!pte_cont(__ptep_get(ptep)))
|
||||
return ptep_clear_flush(vma, addr, ptep);
|
||||
|
||||
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
||||
@ -543,7 +543,7 @@ pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr
|
||||
* when the permission changes from executable to non-executable
|
||||
* in cases where cpu is affected with errata #2645198.
|
||||
*/
|
||||
if (pte_user_exec(ptep_get(ptep)))
|
||||
if (pte_user_exec(__ptep_get(ptep)))
|
||||
return huge_ptep_clear_flush(vma, addr, ptep);
|
||||
}
|
||||
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
|
||||
|
||||
@ -112,8 +112,8 @@ static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
|
||||
if (!early)
|
||||
memset(__va(page_phys), KASAN_SHADOW_INIT, PAGE_SIZE);
|
||||
next = addr + PAGE_SIZE;
|
||||
set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
|
||||
} while (ptep++, addr = next, addr != end && pte_none(ptep_get(ptep)));
|
||||
__set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
|
||||
} while (ptep++, addr = next, addr != end && pte_none(__ptep_get(ptep)));
|
||||
}
|
||||
|
||||
static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
|
||||
@ -271,7 +271,7 @@ static void __init kasan_init_shadow(void)
|
||||
* so we should make sure that it maps the zero page read-only.
|
||||
*/
|
||||
for (i = 0; i < PTRS_PER_PTE; i++)
|
||||
set_pte(&kasan_early_shadow_pte[i],
|
||||
__set_pte(&kasan_early_shadow_pte[i],
|
||||
pfn_pte(sym_to_pfn(kasan_early_shadow_page),
|
||||
PAGE_KERNEL_RO));
|
||||
|
||||
|
||||
@ -173,16 +173,16 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
|
||||
|
||||
ptep = pte_set_fixmap_offset(pmdp, addr);
|
||||
do {
|
||||
pte_t old_pte = ptep_get(ptep);
|
||||
pte_t old_pte = __ptep_get(ptep);
|
||||
|
||||
set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot));
|
||||
__set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot));
|
||||
|
||||
/*
|
||||
* After the PTE entry has been populated once, we
|
||||
* only allow updates to the permission attributes.
|
||||
*/
|
||||
BUG_ON(!pgattr_change_is_safe(pte_val(old_pte),
|
||||
pte_val(ptep_get(ptep))));
|
||||
pte_val(__ptep_get(ptep))));
|
||||
|
||||
phys += PAGE_SIZE;
|
||||
} while (ptep++, addr += PAGE_SIZE, addr != end);
|
||||
@ -852,12 +852,12 @@ static void unmap_hotplug_pte_range(pmd_t *pmdp, unsigned long addr,
|
||||
|
||||
do {
|
||||
ptep = pte_offset_kernel(pmdp, addr);
|
||||
pte = ptep_get(ptep);
|
||||
pte = __ptep_get(ptep);
|
||||
if (pte_none(pte))
|
||||
continue;
|
||||
|
||||
WARN_ON(!pte_present(pte));
|
||||
pte_clear(&init_mm, addr, ptep);
|
||||
__pte_clear(&init_mm, addr, ptep);
|
||||
flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
|
||||
if (free_mapped)
|
||||
free_hotplug_page_range(pte_page(pte),
|
||||
@ -985,7 +985,7 @@ static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
|
||||
|
||||
do {
|
||||
ptep = pte_offset_kernel(pmdp, addr);
|
||||
pte = ptep_get(ptep);
|
||||
pte = __ptep_get(ptep);
|
||||
|
||||
/*
|
||||
* This is just a sanity check here which verifies that
|
||||
@ -1004,7 +1004,7 @@ static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
|
||||
*/
|
||||
ptep = pte_offset_kernel(pmdp, 0UL);
|
||||
for (i = 0; i < PTRS_PER_PTE; i++) {
|
||||
if (!pte_none(ptep_get(&ptep[i])))
|
||||
if (!pte_none(__ptep_get(&ptep[i])))
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@ -36,12 +36,12 @@ bool can_set_direct_map(void)
|
||||
static int change_page_range(pte_t *ptep, unsigned long addr, void *data)
|
||||
{
|
||||
struct page_change_data *cdata = data;
|
||||
pte_t pte = ptep_get(ptep);
|
||||
pte_t pte = __ptep_get(ptep);
|
||||
|
||||
pte = clear_pte_bit(pte, cdata->clear_mask);
|
||||
pte = set_pte_bit(pte, cdata->set_mask);
|
||||
|
||||
set_pte(ptep, pte);
|
||||
__set_pte(ptep, pte);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -245,5 +245,5 @@ bool kernel_page_present(struct page *page)
|
||||
return true;
|
||||
|
||||
ptep = pte_offset_kernel(pmdp, addr);
|
||||
return pte_valid(ptep_get(ptep));
|
||||
return pte_valid(__ptep_get(ptep));
|
||||
}
|
||||
|
||||
@ -33,7 +33,7 @@ static void *trans_alloc(struct trans_pgd_info *info)
|
||||
|
||||
static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr)
|
||||
{
|
||||
pte_t pte = ptep_get(src_ptep);
|
||||
pte_t pte = __ptep_get(src_ptep);
|
||||
|
||||
if (pte_valid(pte)) {
|
||||
/*
|
||||
@ -41,7 +41,7 @@ static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr)
|
||||
* read only (code, rodata). Clear the RDONLY bit from
|
||||
* the temporary mappings we use during restore.
|
||||
*/
|
||||
set_pte(dst_ptep, pte_mkwrite_novma(pte));
|
||||
__set_pte(dst_ptep, pte_mkwrite_novma(pte));
|
||||
} else if ((debug_pagealloc_enabled() ||
|
||||
is_kfence_address((void *)addr)) && !pte_none(pte)) {
|
||||
/*
|
||||
@ -55,7 +55,7 @@ static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr)
|
||||
*/
|
||||
BUG_ON(!pfn_valid(pte_pfn(pte)));
|
||||
|
||||
set_pte(dst_ptep, pte_mkpresent(pte_mkwrite_novma(pte)));
|
||||
__set_pte(dst_ptep, pte_mkpresent(pte_mkwrite_novma(pte)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user