korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-11-11 12:28:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

mm: handle PUD faults under the VMA lock

Browse Source 
Postpone checking the VMA_LOCK flag until we've attempted to handle faults
on PUDs.  There's a mild upside to this patch in that we'll allocate the
page tables while under the VMA lock rather than the mmap lock, reducing
the hold time on the mmap lock, since the retry will find the page tables
already populated.  The real purpose here is to make a commit that shows
we don't call ->huge_fault under the VMA lock.  We do now handle setting
the accessed bit on a PUD fault under the VMA lock, but that doesn't seem
likely to be a measurable difference.

Link: https://lkml.kernel.org/r/20230724185410.1124082-5-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Arjun Roy <arjunroy@google.com>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Punit Agrawal <punit.agrawal@bytedance.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Matthew Wilcox (Oracle) 2023-07-24 19:54:04 +01:00 committed by Andrew Morton
parent 4ec31152a8
commit c4fd825e18
1 changed files with 24 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
mm/memory.c
View File

@ -4859,11 +4859,17 @@ static vm_fault_t create_huge_pud(struct vm_fault *vmf)
{ {
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
struct vm_area_struct *vma = vmf->vma;
/* No support for anonymous transparent PUD pages yet */ /* No support for anonymous transparent PUD pages yet */
if (vma_is_anonymous(vmf->vma)) if (vma_is_anonymous(vma))
return VM_FAULT_FALLBACK; return VM_FAULT_FALLBACK;
if (vmf->vma->vm_ops->huge_fault) if (vma->vm_ops->huge_fault) {
return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD); if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
vma_end_read(vma);
return VM_FAULT_RETRY;
}
return vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
return VM_FAULT_FALLBACK; return VM_FAULT_FALLBACK;
} }
@ -4872,21 +4878,26 @@ static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
{ {
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
struct vm_area_struct *vma = vmf->vma;
vm_fault_t ret; vm_fault_t ret;
/* No support for anonymous transparent PUD pages yet */ /* No support for anonymous transparent PUD pages yet */
if (vma_is_anonymous(vmf->vma)) if (vma_is_anonymous(vma))
goto split; goto split;
if (vmf->vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) { if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
if (vmf->vma->vm_ops->huge_fault) { if (vma->vm_ops->huge_fault) {
ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD); if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
vma_end_read(vma);
return VM_FAULT_RETRY;
}
ret = vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
if (!(ret & VM_FAULT_FALLBACK)) if (!(ret & VM_FAULT_FALLBACK))
return ret; return ret;
} }
} }
split: split:
/* COW or write-notify not handled on PUD level: split pud.*/ /* COW or write-notify not handled on PUD level: split pud.*/
__split_huge_pud(vmf->vma, vmf->pud, vmf->address); __split_huge_pud(vma, vmf->pud, vmf->address);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
return VM_FAULT_FALLBACK; return VM_FAULT_FALLBACK;
} }
@ -5006,11 +5017,6 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
p4d_t *p4d; p4d_t *p4d;
vm_fault_t ret; vm_fault_t ret;
if ((flags & FAULT_FLAG_VMA_LOCK) && !vma_is_anonymous(vma)) {
vma_end_read(vma);
return VM_FAULT_RETRY;
}
pgd = pgd_offset(mm, address); pgd = pgd_offset(mm, address);
p4d = p4d_alloc(mm, pgd, address); p4d = p4d_alloc(mm, pgd, address);
if (!p4d) if (!p4d)
@ -5054,6 +5060,11 @@ retry_pud:
if (pud_trans_unstable(vmf.pud)) if (pud_trans_unstable(vmf.pud))
goto retry_pud; goto retry_pud;
if ((flags & FAULT_FLAG_VMA_LOCK) && !vma_is_anonymous(vma)) {
vma_end_read(vma);
return VM_FAULT_RETRY;
}
if (pmd_none(*vmf.pmd) && if (pmd_none(*vmf.pmd) &&
hugepage_vma_check(vma, vm_flags, false, true, true)) { hugepage_vma_check(vma, vm_flags, false, true, true)) {
ret = create_huge_pmd(&vmf); ret = create_huge_pmd(&vmf);