mirror of
https://github.com/edk2-porting/linux-next.git
synced 2024-12-24 21:24:00 +08:00
KVM/x86: Reduce retpoline performance impact in slot_handle_level_range(), by always inlining iterator helper methods
With retpoline, tight loops of "call this function for every XXX" are very much pessimised by taking a prediction miss *every* time. This one is by far the biggest contributor to the guest launch time with retpoline. By marking the iterator slot_handle_…() functions always_inline, we can ensure that the indirect function call can be optimised away into a direct call and it actually generates slightly smaller code because some of the other conditionals can get optimised away too. Performance is now pretty close to what we see with nospectre_v2 on the command line. Suggested-by: Linus Torvalds <torvalds@linux-foundation.org> Tested-by: Filippo Sironi <sironi@amazon.de> Signed-off-by: David Woodhouse <dwmw@amazon.co.uk> Reviewed-by: Filippo Sironi <sironi@amazon.de> Acked-by: Paolo Bonzini <pbonzini@redhat.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Arjan van de Ven <arjan@linux.intel.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: David Woodhouse <dwmw2@infradead.org> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Josh Poimboeuf <jpoimboe@redhat.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: arjan.van.de.ven@intel.com Cc: dave.hansen@intel.com Cc: jmattson@google.com Cc: karahmed@amazon.de Cc: kvm@vger.kernel.org Cc: rkrcmar@redhat.com Link: http://lkml.kernel.org/r/1518305967-31356-4-git-send-email-dwmw@amazon.co.uk Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
parent
f208820a32
commit
928a4c3948
@ -5058,7 +5058,7 @@ void kvm_mmu_uninit_vm(struct kvm *kvm)
|
||||
typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head);
|
||||
|
||||
/* The caller should hold mmu-lock before calling this function. */
|
||||
static bool
|
||||
static __always_inline bool
|
||||
slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
|
||||
slot_level_handler fn, int start_level, int end_level,
|
||||
gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
|
||||
@ -5088,7 +5088,7 @@ slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
|
||||
return flush;
|
||||
}
|
||||
|
||||
static bool
|
||||
static __always_inline bool
|
||||
slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
|
||||
slot_level_handler fn, int start_level, int end_level,
|
||||
bool lock_flush_tlb)
|
||||
@ -5099,7 +5099,7 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
|
||||
lock_flush_tlb);
|
||||
}
|
||||
|
||||
static bool
|
||||
static __always_inline bool
|
||||
slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
|
||||
slot_level_handler fn, bool lock_flush_tlb)
|
||||
{
|
||||
@ -5107,7 +5107,7 @@ slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
|
||||
PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
|
||||
}
|
||||
|
||||
static bool
|
||||
static __always_inline bool
|
||||
slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
|
||||
slot_level_handler fn, bool lock_flush_tlb)
|
||||
{
|
||||
@ -5115,7 +5115,7 @@ slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
|
||||
PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
|
||||
}
|
||||
|
||||
static bool
|
||||
static __always_inline bool
|
||||
slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
|
||||
slot_level_handler fn, bool lock_flush_tlb)
|
||||
{
|
||||
|
Loading…
Reference in New Issue
Block a user