mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-09 12:04:15 +08:00
d92725256b
I observed that for each of the shared file-backed page faults, we're very likely to retry one more time for the 1st write fault upon no page. It's because we'll need to release the mmap lock for dirty rate limit purpose with balance_dirty_pages_ratelimited() (in fault_dirty_shared_page()). Then after that throttling we return VM_FAULT_RETRY. We did that probably because VM_FAULT_RETRY is the only way we can return to the fault handler at that time telling it we've released the mmap lock. However that's not ideal because it's very likely the fault does not need to be retried at all since the pgtable was well installed before the throttling, so the next continuous fault (including taking mmap read lock, walk the pgtable, etc.) could be in most cases unnecessary. It's not only slowing down page faults for shared file-backed, but also add more mmap lock contention which is in most cases not needed at all. To observe this, one could try to write to some shmem page and look at "pgfault" value in /proc/vmstat, then we should expect 2 counts for each shmem write simply because we retried, and vm event "pgfault" will capture that. To make it more efficient, add a new VM_FAULT_COMPLETED return code just to show that we've completed the whole fault and released the lock. It's also a hint that we should very possibly not need another fault immediately on this page because we've just completed it. This patch provides a ~12% perf boost on my aarch64 test VM with a simple program sequentially dirtying 400MB shmem file being mmap()ed and these are the time it needs: Before: 650.980 ms (+-1.94%) After: 569.396 ms (+-1.38%) I believe it could help more than that. We need some special care on GUP and the s390 pgfault handler (for gmap code before returning from pgfault), the rest changes in the page fault handlers should be relatively straightforward. Another thing to mention is that mm_account_fault() does take this new fault as a generic fault to be accounted, unlike VM_FAULT_RETRY. I explicitly didn't touch hmm_vma_fault() and break_ksm() because they do not handle VM_FAULT_RETRY even with existing code, so I'm literally keeping them as-is. Link: https://lkml.kernel.org/r/20220530183450.42886-1-peterx@redhat.com Signed-off-by: Peter Xu <peterx@redhat.com> Acked-by: Geert Uytterhoeven <geert@linux-m68k.org> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Acked-by: Vineet Gupta <vgupta@kernel.org> Acked-by: Guo Ren <guoren@kernel.org> Acked-by: Max Filippov <jcmvbkbc@gmail.com> Acked-by: Christian Borntraeger <borntraeger@linux.ibm.com> Acked-by: Michael Ellerman <mpe@ellerman.id.au> (powerpc) Acked-by: Catalin Marinas <catalin.marinas@arm.com> Reviewed-by: Alistair Popple <apopple@nvidia.com> Reviewed-by: Ingo Molnar <mingo@kernel.org> Acked-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk> [arm part] Acked-by: Heiko Carstens <hca@linux.ibm.com> Cc: Vasily Gorbik <gor@linux.ibm.com> Cc: Stafford Horne <shorne@gmail.com> Cc: David S. Miller <davem@davemloft.net> Cc: Johannes Berg <johannes@sipsolutions.net> Cc: Brian Cain <bcain@quicinc.com> Cc: Richard Henderson <rth@twiddle.net> Cc: Richard Weinberger <richard@nod.at> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Janosch Frank <frankja@linux.ibm.com> Cc: Albert Ou <aou@eecs.berkeley.edu> Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Sven Schnelle <svens@linux.ibm.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: James Bottomley <James.Bottomley@HansenPartnership.com> Cc: Al Viro <viro@zeniv.linux.org.uk> Cc: Alexander Gordeev <agordeev@linux.ibm.com> Cc: Jonas Bonn <jonas@southpole.se> Cc: Will Deacon <will@kernel.org> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: Michal Simek <monstr@monstr.eu> Cc: Matt Turner <mattst88@gmail.com> Cc: Paul Mackerras <paulus@samba.org> Cc: David Hildenbrand <david@redhat.com> Cc: Nicholas Piggin <npiggin@gmail.com> Cc: Palmer Dabbelt <palmer@dabbelt.com> Cc: Stefan Kristiansson <stefan.kristiansson@saunalahti.fi> Cc: Paul Walmsley <paul.walmsley@sifive.com> Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru> Cc: Chris Zankel <chris@zankel.net> Cc: Hugh Dickins <hughd@google.com> Cc: Dinh Nguyen <dinguyen@kernel.org> Cc: Rich Felker <dalias@libc.org> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de> Cc: Helge Deller <deller@gmx.de> Cc: Yoshinori Sato <ysato@users.osdn.me> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
275 lines
6.5 KiB
C
275 lines
6.5 KiB
C
/*
|
|
* Copyright (C) 2009 Wind River Systems Inc
|
|
* Implemented by fredrik.markstrom@gmail.com and ivarholmqvist@gmail.com
|
|
*
|
|
* based on arch/mips/mm/fault.c which is:
|
|
*
|
|
* Copyright (C) 1995-2000 Ralf Baechle
|
|
*
|
|
* This file is subject to the terms and conditions of the GNU General Public
|
|
* License. See the file "COPYING" in the main directory of this archive
|
|
* for more details.
|
|
*/
|
|
|
|
#include <linux/signal.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/sched/debug.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/string.h>
|
|
#include <linux/types.h>
|
|
#include <linux/ptrace.h>
|
|
#include <linux/mman.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/extable.h>
|
|
#include <linux/uaccess.h>
|
|
#include <linux/perf_event.h>
|
|
|
|
#include <asm/mmu_context.h>
|
|
#include <asm/traps.h>
|
|
|
|
#define EXC_SUPERV_INSN_ACCESS 9 /* Supervisor only instruction address */
|
|
#define EXC_SUPERV_DATA_ACCESS 11 /* Supervisor only data address */
|
|
#define EXC_X_PROTECTION_FAULT 13 /* TLB permission violation (x) */
|
|
#define EXC_R_PROTECTION_FAULT 14 /* TLB permission violation (r) */
|
|
#define EXC_W_PROTECTION_FAULT 15 /* TLB permission violation (w) */
|
|
|
|
/*
|
|
* This routine handles page faults. It determines the address,
|
|
* and the problem, and then passes it off to one of the appropriate
|
|
* routines.
|
|
*/
|
|
asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
|
|
unsigned long address)
|
|
{
|
|
struct vm_area_struct *vma = NULL;
|
|
struct task_struct *tsk = current;
|
|
struct mm_struct *mm = tsk->mm;
|
|
int code = SEGV_MAPERR;
|
|
vm_fault_t fault;
|
|
unsigned int flags = FAULT_FLAG_DEFAULT;
|
|
|
|
cause >>= 2;
|
|
|
|
/* Restart the instruction */
|
|
regs->ea -= 4;
|
|
|
|
/*
|
|
* We fault-in kernel-space virtual memory on-demand. The
|
|
* 'reference' page table is init_mm.pgd.
|
|
*
|
|
* NOTE! We MUST NOT take any locks for this case. We may
|
|
* be in an interrupt or a critical region, and should
|
|
* only copy the information from the master page table,
|
|
* nothing more.
|
|
*/
|
|
if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END)) {
|
|
if (user_mode(regs))
|
|
goto bad_area_nosemaphore;
|
|
else
|
|
goto vmalloc_fault;
|
|
}
|
|
|
|
if (unlikely(address >= TASK_SIZE))
|
|
goto bad_area_nosemaphore;
|
|
|
|
/*
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
if (faulthandler_disabled() || !mm)
|
|
goto bad_area_nosemaphore;
|
|
|
|
if (user_mode(regs))
|
|
flags |= FAULT_FLAG_USER;
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
|
|
|
|
if (!mmap_read_trylock(mm)) {
|
|
if (!user_mode(regs) && !search_exception_tables(regs->ea))
|
|
goto bad_area_nosemaphore;
|
|
retry:
|
|
mmap_read_lock(mm);
|
|
}
|
|
|
|
vma = find_vma(mm, address);
|
|
if (!vma)
|
|
goto bad_area;
|
|
if (vma->vm_start <= address)
|
|
goto good_area;
|
|
if (!(vma->vm_flags & VM_GROWSDOWN))
|
|
goto bad_area;
|
|
if (expand_stack(vma, address))
|
|
goto bad_area;
|
|
/*
|
|
* Ok, we have a good vm_area for this memory access, so
|
|
* we can handle it..
|
|
*/
|
|
good_area:
|
|
code = SEGV_ACCERR;
|
|
|
|
switch (cause) {
|
|
case EXC_SUPERV_INSN_ACCESS:
|
|
goto bad_area;
|
|
case EXC_SUPERV_DATA_ACCESS:
|
|
goto bad_area;
|
|
case EXC_X_PROTECTION_FAULT:
|
|
if (!(vma->vm_flags & VM_EXEC))
|
|
goto bad_area;
|
|
break;
|
|
case EXC_R_PROTECTION_FAULT:
|
|
if (!(vma->vm_flags & VM_READ))
|
|
goto bad_area;
|
|
break;
|
|
case EXC_W_PROTECTION_FAULT:
|
|
if (!(vma->vm_flags & VM_WRITE))
|
|
goto bad_area;
|
|
flags = FAULT_FLAG_WRITE;
|
|
break;
|
|
}
|
|
|
|
/*
|
|
* If for any reason at all we couldn't handle the fault,
|
|
* make sure we exit gracefully rather than endlessly redo
|
|
* the fault.
|
|
*/
|
|
fault = handle_mm_fault(vma, address, flags, regs);
|
|
|
|
if (fault_signal_pending(fault, regs))
|
|
return;
|
|
|
|
/* The fault is fully completed (including releasing mmap lock) */
|
|
if (fault & VM_FAULT_COMPLETED)
|
|
return;
|
|
|
|
if (unlikely(fault & VM_FAULT_ERROR)) {
|
|
if (fault & VM_FAULT_OOM)
|
|
goto out_of_memory;
|
|
else if (fault & VM_FAULT_SIGSEGV)
|
|
goto bad_area;
|
|
else if (fault & VM_FAULT_SIGBUS)
|
|
goto do_sigbus;
|
|
BUG();
|
|
}
|
|
|
|
if (fault & VM_FAULT_RETRY) {
|
|
flags |= FAULT_FLAG_TRIED;
|
|
|
|
/*
|
|
* No need to mmap_read_unlock(mm) as we would
|
|
* have already released it in __lock_page_or_retry
|
|
* in mm/filemap.c.
|
|
*/
|
|
|
|
goto retry;
|
|
}
|
|
|
|
mmap_read_unlock(mm);
|
|
return;
|
|
|
|
/*
|
|
* Something tried to access memory that isn't in our memory map..
|
|
* Fix it, but check if it's kernel or user first..
|
|
*/
|
|
bad_area:
|
|
mmap_read_unlock(mm);
|
|
|
|
bad_area_nosemaphore:
|
|
/* User mode accesses just cause a SIGSEGV */
|
|
if (user_mode(regs)) {
|
|
if (unhandled_signal(current, SIGSEGV) && printk_ratelimit()) {
|
|
pr_info("%s: unhandled page fault (%d) at 0x%08lx, "
|
|
"cause %ld\n", current->comm, SIGSEGV, address, cause);
|
|
show_regs(regs);
|
|
}
|
|
_exception(SIGSEGV, regs, code, address);
|
|
return;
|
|
}
|
|
|
|
no_context:
|
|
/* Are we prepared to handle this kernel fault? */
|
|
if (fixup_exception(regs))
|
|
return;
|
|
|
|
/*
|
|
* Oops. The kernel tried to access some bad page. We'll have to
|
|
* terminate things with extreme prejudice.
|
|
*/
|
|
bust_spinlocks(1);
|
|
|
|
pr_alert("Unable to handle kernel %s at virtual address %08lx",
|
|
address < PAGE_SIZE ? "NULL pointer dereference" :
|
|
"paging request", address);
|
|
pr_alert("ea = %08lx, ra = %08lx, cause = %ld\n", regs->ea, regs->ra,
|
|
cause);
|
|
panic("Oops");
|
|
return;
|
|
|
|
/*
|
|
* We ran out of memory, or some other thing happened to us that made
|
|
* us unable to handle the page fault gracefully.
|
|
*/
|
|
out_of_memory:
|
|
mmap_read_unlock(mm);
|
|
if (!user_mode(regs))
|
|
goto no_context;
|
|
pagefault_out_of_memory();
|
|
return;
|
|
|
|
do_sigbus:
|
|
mmap_read_unlock(mm);
|
|
|
|
/* Kernel mode? Handle exceptions or die */
|
|
if (!user_mode(regs))
|
|
goto no_context;
|
|
|
|
_exception(SIGBUS, regs, BUS_ADRERR, address);
|
|
return;
|
|
|
|
vmalloc_fault:
|
|
{
|
|
/*
|
|
* Synchronize this task's top level page-table
|
|
* with the 'reference' page table.
|
|
*
|
|
* Do _not_ use "tsk" here. We might be inside
|
|
* an interrupt in the middle of a task switch..
|
|
*/
|
|
int offset = pgd_index(address);
|
|
pgd_t *pgd, *pgd_k;
|
|
p4d_t *p4d, *p4d_k;
|
|
pud_t *pud, *pud_k;
|
|
pmd_t *pmd, *pmd_k;
|
|
pte_t *pte_k;
|
|
|
|
pgd = pgd_current + offset;
|
|
pgd_k = init_mm.pgd + offset;
|
|
|
|
if (!pgd_present(*pgd_k))
|
|
goto no_context;
|
|
set_pgd(pgd, *pgd_k);
|
|
|
|
p4d = p4d_offset(pgd, address);
|
|
p4d_k = p4d_offset(pgd_k, address);
|
|
if (!p4d_present(*p4d_k))
|
|
goto no_context;
|
|
pud = pud_offset(p4d, address);
|
|
pud_k = pud_offset(p4d_k, address);
|
|
if (!pud_present(*pud_k))
|
|
goto no_context;
|
|
pmd = pmd_offset(pud, address);
|
|
pmd_k = pmd_offset(pud_k, address);
|
|
if (!pmd_present(*pmd_k))
|
|
goto no_context;
|
|
set_pmd(pmd, *pmd_k);
|
|
|
|
pte_k = pte_offset_kernel(pmd_k, address);
|
|
if (!pte_present(*pte_k))
|
|
goto no_context;
|
|
|
|
flush_tlb_kernel_page(address);
|
|
return;
|
|
}
|
|
}
|