2012-12-11 00:29:28 +08:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2012,2013 - ARM Ltd
|
|
|
|
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
|
|
|
*
|
|
|
|
* Derived from arch/arm/include/asm/kvm_host.h:
|
|
|
|
* Copyright (C) 2012 - Virtual Open Systems and Columbia University
|
|
|
|
* Author: Christoffer Dall <c.dall@virtualopensystems.com>
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef __ARM64_KVM_HOST_H__
|
|
|
|
#define __ARM64_KVM_HOST_H__
|
|
|
|
|
2014-08-29 20:01:17 +08:00
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/kvm_types.h>
|
2012-12-11 00:29:28 +08:00
|
|
|
#include <asm/kvm.h>
|
2015-01-29 21:19:45 +08:00
|
|
|
#include <asm/kvm_asm.h>
|
2012-12-11 00:29:28 +08:00
|
|
|
#include <asm/kvm_mmio.h>
|
|
|
|
|
2015-03-04 18:14:34 +08:00
|
|
|
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
|
|
|
|
|
2012-12-11 00:29:28 +08:00
|
|
|
#define KVM_USER_MEM_SLOTS 32
|
|
|
|
#define KVM_PRIVATE_MEM_SLOTS 4
|
|
|
|
#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
|
2015-09-18 18:34:53 +08:00
|
|
|
#define KVM_HALT_POLL_NS_DEFAULT 500000
|
2012-12-11 00:29:28 +08:00
|
|
|
|
|
|
|
#include <kvm/arm_vgic.h>
|
|
|
|
#include <kvm/arm_arch_timer.h>
|
2015-09-11 09:38:32 +08:00
|
|
|
#include <kvm/arm_pmu.h>
|
2012-12-11 00:29:28 +08:00
|
|
|
|
arm/arm64: KVM: Remove 'config KVM_ARM_MAX_VCPUS'
This patch removes config option of KVM_ARM_MAX_VCPUS,
and like other ARCHs, just choose the maximum allowed
value from hardware, and follows the reasons:
1) from distribution view, the option has to be
defined as the max allowed value because it need to
meet all kinds of virtulization applications and
need to support most of SoCs;
2) using a bigger value doesn't introduce extra memory
consumption, and the help text in Kconfig isn't accurate
because kvm_vpu structure isn't allocated until request
of creating VCPU is sent from QEMU;
3) the main effect is that the field of vcpus[] in 'struct kvm'
becomes a bit bigger(sizeof(void *) per vcpu) and need more cache
lines to hold the structure, but 'struct kvm' is one generic struct,
and it has worked well on other ARCHs already in this way. Also,
the world switch frequecy is often low, for example, it is ~2000
when running kernel building load in VM from APM xgene KVM host,
so the effect is very small, and the difference can't be observed
in my test at all.
Cc: Dann Frazier <dann.frazier@canonical.com>
Signed-off-by: Ming Lei <ming.lei@canonical.com>
Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2015-09-02 14:31:21 +08:00
|
|
|
#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
|
|
|
|
|
2016-01-11 22:46:15 +08:00
|
|
|
#define KVM_VCPU_MAX_FEATURES 4
|
2012-12-11 00:29:28 +08:00
|
|
|
|
2016-04-27 17:28:00 +08:00
|
|
|
#define KVM_REQ_VCPU_EXIT 8
|
|
|
|
|
2014-08-26 22:13:20 +08:00
|
|
|
int __attribute_const__ kvm_target_cpu(void);
|
2012-12-11 00:29:28 +08:00
|
|
|
int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
|
|
|
|
int kvm_arch_dev_ioctl_check_extension(long ext);
|
arm64: kvm: Fix kvm teardown for systems using the extended idmap
If memory is located above 1<<VA_BITS, kvm adds an extra level to its page
tables, merging the runtime tables and boot tables that contain the idmap.
This lets us avoid the trampoline dance during initialisation.
This also means there is no trampoline page mapped, so
__cpu_reset_hyp_mode() can't call __kvm_hyp_reset() in this page. The good
news is the idmap is still mapped, so we don't need the trampoline page.
The bad news is we can't call it directly as the idmap is above
HYP_PAGE_OFFSET, so its address is masked by kvm_call_hyp.
Add a function __extended_idmap_trampoline which will branch into
__kvm_hyp_reset in the idmap, change kvm_hyp_reset_entry() to return
this address if __kvm_cpu_uses_extended_idmap(). In this case
__kvm_hyp_reset() will still switch to the boot tables (which are the
merged tables that were already in use), and branch into the idmap (where
it already was).
This fixes boot failures on these systems, where we fail to execute the
missing trampoline page when tearing down kvm in init_subsystems():
[ 2.508922] kvm [1]: 8-bit VMID
[ 2.512057] kvm [1]: Hyp mode initialized successfully
[ 2.517242] kvm [1]: interrupt-controller@e1140000 IRQ13
[ 2.522622] kvm [1]: timer IRQ3
[ 2.525783] Kernel panic - not syncing: HYP panic:
[ 2.525783] PS:200003c9 PC:0000007ffffff820 ESR:86000005
[ 2.525783] FAR:0000007ffffff820 HPFAR:00000000003ffff0 PAR:0000000000000000
[ 2.525783] VCPU: (null)
[ 2.525783]
[ 2.547667] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 4.6.0-rc5+ #1
[ 2.555137] Hardware name: Default string Default string/Default string, BIOS ROD0084E 09/03/2015
[ 2.563994] Call trace:
[ 2.566432] [<ffffff80080888d0>] dump_backtrace+0x0/0x240
[ 2.571818] [<ffffff8008088b24>] show_stack+0x14/0x20
[ 2.576858] [<ffffff80083423ac>] dump_stack+0x94/0xb8
[ 2.581899] [<ffffff8008152130>] panic+0x10c/0x250
[ 2.586677] [<ffffff8008152024>] panic+0x0/0x250
[ 2.591281] SMP: stopping secondary CPUs
[ 3.649692] SMP: failed to stop secondary CPUs 0-2,4-7
[ 3.654818] Kernel Offset: disabled
[ 3.658293] Memory Limit: none
[ 3.661337] ---[ end Kernel panic - not syncing: HYP panic:
[ 3.661337] PS:200003c9 PC:0000007ffffff820 ESR:86000005
[ 3.661337] FAR:0000007ffffff820 HPFAR:00000000003ffff0 PAR:0000000000000000
[ 3.661337] VCPU: (null)
[ 3.661337]
Reported-by: Will Deacon <will.deacon@arm.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2016-04-30 01:27:03 +08:00
|
|
|
void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start);
|
2012-12-11 00:29:28 +08:00
|
|
|
|
|
|
|
struct kvm_arch {
|
|
|
|
/* The VMID generation used for the virt. memory system */
|
|
|
|
u64 vmid_gen;
|
|
|
|
u32 vmid;
|
|
|
|
|
|
|
|
/* 1-level 2nd stage table and lock */
|
|
|
|
spinlock_t pgd_lock;
|
|
|
|
pgd_t *pgd;
|
|
|
|
|
|
|
|
/* VTTBR value associated with above pgd and vmid */
|
|
|
|
u64 vttbr;
|
|
|
|
|
2014-06-02 22:26:01 +08:00
|
|
|
/* The maximum number of vCPUs depends on the used GIC model */
|
|
|
|
int max_vcpus;
|
|
|
|
|
2012-12-11 00:29:28 +08:00
|
|
|
/* Interrupt controller */
|
|
|
|
struct vgic_dist vgic;
|
|
|
|
|
|
|
|
/* Timer */
|
|
|
|
struct arch_timer_kvm timer;
|
|
|
|
};
|
|
|
|
|
|
|
|
#define KVM_NR_MEM_OBJS 40
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't want allocation failures within the mmu code, so we preallocate
|
|
|
|
* enough memory for a single page fault in a cache.
|
|
|
|
*/
|
|
|
|
struct kvm_mmu_memory_cache {
|
|
|
|
int nobjs;
|
|
|
|
void *objects[KVM_NR_MEM_OBJS];
|
|
|
|
};
|
|
|
|
|
|
|
|
struct kvm_vcpu_fault_info {
|
|
|
|
u32 esr_el2; /* Hyp Syndrom Register */
|
|
|
|
u64 far_el2; /* Hyp Fault Address Register */
|
|
|
|
u64 hpfar_el2; /* Hyp IPA Fault Address Register */
|
|
|
|
};
|
|
|
|
|
2015-10-26 03:57:11 +08:00
|
|
|
/*
|
|
|
|
* 0 is reserved as an invalid value.
|
|
|
|
* Order should be kept in sync with the save/restore code.
|
|
|
|
*/
|
|
|
|
enum vcpu_sysreg {
|
|
|
|
__INVALID_SYSREG__,
|
|
|
|
MPIDR_EL1, /* MultiProcessor Affinity Register */
|
|
|
|
CSSELR_EL1, /* Cache Size Selection Register */
|
|
|
|
SCTLR_EL1, /* System Control Register */
|
|
|
|
ACTLR_EL1, /* Auxiliary Control Register */
|
|
|
|
CPACR_EL1, /* Coprocessor Access Control */
|
|
|
|
TTBR0_EL1, /* Translation Table Base Register 0 */
|
|
|
|
TTBR1_EL1, /* Translation Table Base Register 1 */
|
|
|
|
TCR_EL1, /* Translation Control Register */
|
|
|
|
ESR_EL1, /* Exception Syndrome Register */
|
2016-02-25 01:52:41 +08:00
|
|
|
AFSR0_EL1, /* Auxiliary Fault Status Register 0 */
|
|
|
|
AFSR1_EL1, /* Auxiliary Fault Status Register 1 */
|
2015-10-26 03:57:11 +08:00
|
|
|
FAR_EL1, /* Fault Address Register */
|
|
|
|
MAIR_EL1, /* Memory Attribute Indirection Register */
|
|
|
|
VBAR_EL1, /* Vector Base Address Register */
|
|
|
|
CONTEXTIDR_EL1, /* Context ID Register */
|
|
|
|
TPIDR_EL0, /* Thread ID, User R/W */
|
|
|
|
TPIDRRO_EL0, /* Thread ID, User R/O */
|
|
|
|
TPIDR_EL1, /* Thread ID, Privileged */
|
|
|
|
AMAIR_EL1, /* Aux Memory Attribute Indirection Register */
|
|
|
|
CNTKCTL_EL1, /* Timer Control Register (EL1) */
|
|
|
|
PAR_EL1, /* Physical Address Register */
|
|
|
|
MDSCR_EL1, /* Monitor Debug System Control Register */
|
|
|
|
MDCCINT_EL1, /* Monitor Debug Comms Channel Interrupt Enable Reg */
|
|
|
|
|
2015-06-18 16:01:53 +08:00
|
|
|
/* Performance Monitors Registers */
|
|
|
|
PMCR_EL0, /* Control Register */
|
2015-08-31 17:20:22 +08:00
|
|
|
PMSELR_EL0, /* Event Counter Selection Register */
|
2015-12-08 15:29:06 +08:00
|
|
|
PMEVCNTR0_EL0, /* Event Counter Register (0-30) */
|
|
|
|
PMEVCNTR30_EL0 = PMEVCNTR0_EL0 + 30,
|
|
|
|
PMCCNTR_EL0, /* Cycle Counter Register */
|
2016-02-23 11:11:27 +08:00
|
|
|
PMEVTYPER0_EL0, /* Event Type Register (0-30) */
|
|
|
|
PMEVTYPER30_EL0 = PMEVTYPER0_EL0 + 30,
|
|
|
|
PMCCFILTR_EL0, /* Cycle Count Filter Register */
|
2015-09-08 12:26:13 +08:00
|
|
|
PMCNTENSET_EL0, /* Count Enable Set Register */
|
2015-09-08 14:40:20 +08:00
|
|
|
PMINTENSET_EL1, /* Interrupt Enable Set Register */
|
2015-09-08 15:03:26 +08:00
|
|
|
PMOVSSET_EL0, /* Overflow Flag Status Set Register */
|
2015-09-08 15:49:39 +08:00
|
|
|
PMSWINC_EL0, /* Software Increment Register */
|
2015-09-08 15:15:56 +08:00
|
|
|
PMUSERENR_EL0, /* User Enable Register */
|
2015-06-18 16:01:53 +08:00
|
|
|
|
2015-10-26 03:57:11 +08:00
|
|
|
/* 32bit specific registers. Keep them at the end of the range */
|
|
|
|
DACR32_EL2, /* Domain Access Control Register */
|
|
|
|
IFSR32_EL2, /* Instruction Fault Status Register */
|
|
|
|
FPEXC32_EL2, /* Floating-Point Exception Control Register */
|
|
|
|
DBGVCR32_EL2, /* Debug Vector Catch Register */
|
|
|
|
|
|
|
|
NR_SYS_REGS /* Nothing after this line! */
|
|
|
|
};
|
|
|
|
|
|
|
|
/* 32bit mapping */
|
|
|
|
#define c0_MPIDR (MPIDR_EL1 * 2) /* MultiProcessor ID Register */
|
|
|
|
#define c0_CSSELR (CSSELR_EL1 * 2)/* Cache Size Selection Register */
|
|
|
|
#define c1_SCTLR (SCTLR_EL1 * 2) /* System Control Register */
|
|
|
|
#define c1_ACTLR (ACTLR_EL1 * 2) /* Auxiliary Control Register */
|
|
|
|
#define c1_CPACR (CPACR_EL1 * 2) /* Coprocessor Access Control */
|
|
|
|
#define c2_TTBR0 (TTBR0_EL1 * 2) /* Translation Table Base Register 0 */
|
|
|
|
#define c2_TTBR0_high (c2_TTBR0 + 1) /* TTBR0 top 32 bits */
|
|
|
|
#define c2_TTBR1 (TTBR1_EL1 * 2) /* Translation Table Base Register 1 */
|
|
|
|
#define c2_TTBR1_high (c2_TTBR1 + 1) /* TTBR1 top 32 bits */
|
|
|
|
#define c2_TTBCR (TCR_EL1 * 2) /* Translation Table Base Control R. */
|
|
|
|
#define c3_DACR (DACR32_EL2 * 2)/* Domain Access Control Register */
|
|
|
|
#define c5_DFSR (ESR_EL1 * 2) /* Data Fault Status Register */
|
|
|
|
#define c5_IFSR (IFSR32_EL2 * 2)/* Instruction Fault Status Register */
|
|
|
|
#define c5_ADFSR (AFSR0_EL1 * 2) /* Auxiliary Data Fault Status R */
|
|
|
|
#define c5_AIFSR (AFSR1_EL1 * 2) /* Auxiliary Instr Fault Status R */
|
|
|
|
#define c6_DFAR (FAR_EL1 * 2) /* Data Fault Address Register */
|
|
|
|
#define c6_IFAR (c6_DFAR + 1) /* Instruction Fault Address Register */
|
|
|
|
#define c7_PAR (PAR_EL1 * 2) /* Physical Address Register */
|
|
|
|
#define c7_PAR_high (c7_PAR + 1) /* PAR top 32 bits */
|
|
|
|
#define c10_PRRR (MAIR_EL1 * 2) /* Primary Region Remap Register */
|
|
|
|
#define c10_NMRR (c10_PRRR + 1) /* Normal Memory Remap Register */
|
|
|
|
#define c12_VBAR (VBAR_EL1 * 2) /* Vector Base Address Register */
|
|
|
|
#define c13_CID (CONTEXTIDR_EL1 * 2) /* Context ID Register */
|
|
|
|
#define c13_TID_URW (TPIDR_EL0 * 2) /* Thread ID, User R/W */
|
|
|
|
#define c13_TID_URO (TPIDRRO_EL0 * 2)/* Thread ID, User R/O */
|
|
|
|
#define c13_TID_PRIV (TPIDR_EL1 * 2) /* Thread ID, Privileged */
|
|
|
|
#define c10_AMAIR0 (AMAIR_EL1 * 2) /* Aux Memory Attr Indirection Reg */
|
|
|
|
#define c10_AMAIR1 (c10_AMAIR0 + 1)/* Aux Memory Attr Indirection Reg */
|
|
|
|
#define c14_CNTKCTL (CNTKCTL_EL1 * 2) /* Timer Control Register (PL1) */
|
|
|
|
|
|
|
|
#define cp14_DBGDSCRext (MDSCR_EL1 * 2)
|
|
|
|
#define cp14_DBGBCR0 (DBGBCR0_EL1 * 2)
|
|
|
|
#define cp14_DBGBVR0 (DBGBVR0_EL1 * 2)
|
|
|
|
#define cp14_DBGBXVR0 (cp14_DBGBVR0 + 1)
|
|
|
|
#define cp14_DBGWCR0 (DBGWCR0_EL1 * 2)
|
|
|
|
#define cp14_DBGWVR0 (DBGWVR0_EL1 * 2)
|
|
|
|
#define cp14_DBGDCCINT (MDCCINT_EL1 * 2)
|
|
|
|
|
|
|
|
#define NR_COPRO_REGS (NR_SYS_REGS * 2)
|
|
|
|
|
2012-12-11 00:29:28 +08:00
|
|
|
struct kvm_cpu_context {
|
|
|
|
struct kvm_regs gp_regs;
|
2013-02-07 03:17:50 +08:00
|
|
|
union {
|
|
|
|
u64 sys_regs[NR_SYS_REGS];
|
2014-04-24 17:27:13 +08:00
|
|
|
u32 copro[NR_COPRO_REGS];
|
2013-02-07 03:17:50 +08:00
|
|
|
};
|
2012-12-11 00:29:28 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
typedef struct kvm_cpu_context kvm_cpu_context_t;
|
|
|
|
|
|
|
|
struct kvm_vcpu_arch {
|
|
|
|
struct kvm_cpu_context ctxt;
|
|
|
|
|
|
|
|
/* HYP configuration */
|
|
|
|
u64 hcr_el2;
|
2015-07-08 00:29:56 +08:00
|
|
|
u32 mdcr_el2;
|
2012-12-11 00:29:28 +08:00
|
|
|
|
|
|
|
/* Exception Information */
|
|
|
|
struct kvm_vcpu_fault_info fault;
|
|
|
|
|
2015-07-08 00:30:00 +08:00
|
|
|
/* Guest debug state */
|
2014-04-24 17:24:46 +08:00
|
|
|
u64 debug_flags;
|
|
|
|
|
2015-07-08 00:30:00 +08:00
|
|
|
/*
|
|
|
|
* We maintain more than a single set of debug registers to support
|
|
|
|
* debugging the guest from the host and to maintain separate host and
|
|
|
|
* guest state during world switches. vcpu_debug_state are the debug
|
|
|
|
* registers of the vcpu as the guest sees them. host_debug_state are
|
2015-07-08 00:30:02 +08:00
|
|
|
* the host registers which are saved and restored during
|
|
|
|
* world switches. external_debug_state contains the debug
|
|
|
|
* values we want to debug the guest. This is set via the
|
|
|
|
* KVM_SET_GUEST_DEBUG ioctl.
|
2015-07-08 00:30:00 +08:00
|
|
|
*
|
|
|
|
* debug_ptr points to the set of debug registers that should be loaded
|
|
|
|
* onto the hardware when running the guest.
|
|
|
|
*/
|
|
|
|
struct kvm_guest_debug_arch *debug_ptr;
|
|
|
|
struct kvm_guest_debug_arch vcpu_debug_state;
|
2015-07-08 00:30:02 +08:00
|
|
|
struct kvm_guest_debug_arch external_debug_state;
|
2015-07-08 00:30:00 +08:00
|
|
|
|
2012-12-11 00:29:28 +08:00
|
|
|
/* Pointer to host CPU context */
|
|
|
|
kvm_cpu_context_t *host_cpu_context;
|
2015-07-08 00:30:00 +08:00
|
|
|
struct kvm_guest_debug_arch host_debug_state;
|
2012-12-11 00:29:28 +08:00
|
|
|
|
|
|
|
/* VGIC state */
|
|
|
|
struct vgic_cpu vgic_cpu;
|
|
|
|
struct arch_timer_cpu timer_cpu;
|
2015-09-11 09:38:32 +08:00
|
|
|
struct kvm_pmu pmu;
|
2012-12-11 00:29:28 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Anything that is not used directly from assembly code goes
|
|
|
|
* here.
|
|
|
|
*/
|
|
|
|
|
2015-07-08 00:29:58 +08:00
|
|
|
/*
|
|
|
|
* Guest registers we preserve during guest debugging.
|
|
|
|
*
|
|
|
|
* These shadow registers are updated by the kvm_handle_sys_reg
|
|
|
|
* trap handler if the guest accesses or updates them while we
|
|
|
|
* are using guest debug.
|
|
|
|
*/
|
|
|
|
struct {
|
|
|
|
u32 mdscr_el1;
|
|
|
|
} guest_debug_preserved;
|
|
|
|
|
2015-09-26 05:41:14 +08:00
|
|
|
/* vcpu power-off state */
|
|
|
|
bool power_off;
|
2012-12-11 00:29:28 +08:00
|
|
|
|
2015-09-26 05:41:17 +08:00
|
|
|
/* Don't run the guest (internal implementation need) */
|
|
|
|
bool pause;
|
|
|
|
|
2012-12-11 00:29:28 +08:00
|
|
|
/* IO related fields */
|
|
|
|
struct kvm_decode mmio_decode;
|
|
|
|
|
|
|
|
/* Interrupt related fields */
|
|
|
|
u64 irq_lines; /* IRQ and FIQ levels */
|
|
|
|
|
|
|
|
/* Cache some mmu pages needed inside spinlock regions */
|
|
|
|
struct kvm_mmu_memory_cache mmu_page_cache;
|
|
|
|
|
|
|
|
/* Target CPU and feature flags */
|
2013-07-22 11:40:38 +08:00
|
|
|
int target;
|
2012-12-11 00:29:28 +08:00
|
|
|
DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
|
|
|
|
|
|
|
|
/* Detect first run of a vcpu */
|
|
|
|
bool has_run_once;
|
|
|
|
};
|
|
|
|
|
|
|
|
#define vcpu_gp_regs(v) (&(v)->arch.ctxt.gp_regs)
|
|
|
|
#define vcpu_sys_reg(v,r) ((v)->arch.ctxt.sys_regs[(r)])
|
2014-04-24 17:27:13 +08:00
|
|
|
/*
|
|
|
|
* CP14 and CP15 live in the same array, as they are backed by the
|
|
|
|
* same system registers.
|
|
|
|
*/
|
|
|
|
#define vcpu_cp14(v,r) ((v)->arch.ctxt.copro[(r)])
|
|
|
|
#define vcpu_cp15(v,r) ((v)->arch.ctxt.copro[(r)])
|
2012-12-11 00:29:28 +08:00
|
|
|
|
2014-07-03 00:19:30 +08:00
|
|
|
#ifdef CONFIG_CPU_BIG_ENDIAN
|
2014-08-01 19:00:36 +08:00
|
|
|
#define vcpu_cp15_64_high(v,r) vcpu_cp15((v),(r))
|
|
|
|
#define vcpu_cp15_64_low(v,r) vcpu_cp15((v),(r) + 1)
|
2014-07-03 00:19:30 +08:00
|
|
|
#else
|
2014-08-01 19:00:36 +08:00
|
|
|
#define vcpu_cp15_64_high(v,r) vcpu_cp15((v),(r) + 1)
|
|
|
|
#define vcpu_cp15_64_low(v,r) vcpu_cp15((v),(r))
|
2014-07-03 00:19:30 +08:00
|
|
|
#endif
|
|
|
|
|
2012-12-11 00:29:28 +08:00
|
|
|
struct kvm_vm_stat {
|
|
|
|
u32 remote_tlb_flush;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct kvm_vcpu_stat {
|
kvm: add halt_poll_ns module parameter
This patch introduces a new module parameter for the KVM module; when it
is present, KVM attempts a bit of polling on every HLT before scheduling
itself out via kvm_vcpu_block.
This parameter helps a lot for latency-bound workloads---in particular
I tested it with O_DSYNC writes with a battery-backed disk in the host.
In this case, writes are fast (because the data doesn't have to go all
the way to the platters) but they cannot be merged by either the host or
the guest. KVM's performance here is usually around 30% of bare metal,
or 50% if you use cache=directsync or cache=writethrough (these
parameters avoid that the guest sends pointless flush requests, and
at the same time they are not slow because of the battery-backed cache).
The bad performance happens because on every halt the host CPU decides
to halt itself too. When the interrupt comes, the vCPU thread is then
migrated to a new physical CPU, and in general the latency is horrible
because the vCPU thread has to be scheduled back in.
With this patch performance reaches 60-65% of bare metal and, more
important, 99% of what you get if you use idle=poll in the guest. This
means that the tunable gets rid of this particular bottleneck, and more
work can be done to improve performance in the kernel or QEMU.
Of course there is some price to pay; every time an otherwise idle vCPUs
is interrupted by an interrupt, it will poll unnecessarily and thus
impose a little load on the host. The above results were obtained with
a mostly random value of the parameter (500000), and the load was around
1.5-2.5% CPU usage on one of the host's core for each idle guest vCPU.
The patch also adds a new stat, /sys/kernel/debug/kvm/halt_successful_poll,
that can be used to tune the parameter. It counts how many HLT
instructions received an interrupt during the polling period; each
successful poll avoids that Linux schedules the VCPU thread out and back
in, and may also avoid a likely trip to C1 and back for the physical CPU.
While the VM is idle, a Linux 4 VCPU VM halts around 10 times per second.
Of these halts, almost all are failed polls. During the benchmark,
instead, basically all halts end within the polling period, except a more
or less constant stream of 50 per second coming from vCPUs that are not
running the benchmark. The wasted time is thus very low. Things may
be slightly different for Windows VMs, which have a ~10 ms timer tick.
The effect is also visible on Marcelo's recently-introduced latency
test for the TSC deadline timer. Though of course a non-RT kernel has
awful latency bounds, the latency of the timer is around 8000-10000 clock
cycles compared to 20000-120000 without setting halt_poll_ns. For the TSC
deadline timer, thus, the effect is both a smaller average latency and
a smaller variance.
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2015-02-05 01:20:58 +08:00
|
|
|
u32 halt_successful_poll;
|
2015-09-16 00:27:57 +08:00
|
|
|
u32 halt_attempted_poll;
|
2016-05-13 18:16:35 +08:00
|
|
|
u32 halt_poll_invalid;
|
2012-12-11 00:29:28 +08:00
|
|
|
u32 halt_wakeup;
|
2015-11-26 18:09:43 +08:00
|
|
|
u32 hvc_exit_stat;
|
|
|
|
u64 wfe_exit_stat;
|
|
|
|
u64 wfi_exit_stat;
|
|
|
|
u64 mmio_exit_user;
|
|
|
|
u64 mmio_exit_kernel;
|
|
|
|
u64 exits;
|
2012-12-11 00:29:28 +08:00
|
|
|
};
|
|
|
|
|
2013-09-30 16:50:06 +08:00
|
|
|
int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
|
2012-12-11 00:29:28 +08:00
|
|
|
unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
|
|
|
|
int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
|
|
|
|
int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
|
|
|
|
int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
|
|
|
|
|
|
|
|
#define KVM_ARCH_WANT_MMU_NOTIFIER
|
|
|
|
int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
|
|
|
|
int kvm_unmap_hva_range(struct kvm *kvm,
|
|
|
|
unsigned long start, unsigned long end);
|
|
|
|
void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
|
2015-03-13 02:16:51 +08:00
|
|
|
int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
|
|
|
|
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
|
2012-12-11 00:29:28 +08:00
|
|
|
|
|
|
|
/* We do not have shadow page tables, hence the empty hooks */
|
2014-09-24 15:57:57 +08:00
|
|
|
static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
|
|
|
|
unsigned long address)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2012-12-11 00:29:28 +08:00
|
|
|
struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
|
2014-08-26 22:13:21 +08:00
|
|
|
struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
|
2016-04-27 17:28:00 +08:00
|
|
|
void kvm_arm_halt_guest(struct kvm *kvm);
|
|
|
|
void kvm_arm_resume_guest(struct kvm *kvm);
|
KVM: arm/arm64: vgic-new: Synchronize changes to active state
When modifying the active state of an interrupt via the MMIO interface,
we should ensure that the write has the intended effect.
If a guest sets an interrupt to active, but that interrupt is already
flushed into a list register on a running VCPU, then that VCPU will
write the active state back into the struct vgic_irq upon returning from
the guest and syncing its state. This is a non-benign race, because the
guest can observe that an interrupt is not active, and it can have a
reasonable expectations that other VCPUs will not ack any IRQs, and then
set the state to active, and expect it to stay that way. Currently we
are not honoring this case.
Thefore, change both the SACTIVE and CACTIVE mmio handlers to stop the
world, change the irq state, potentially queue the irq if we're setting
it to active, and then continue.
We take this chance to slightly optimize these functions by not stopping
the world when touching private interrupts where there is inherently no
possible race.
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2016-05-20 21:25:28 +08:00
|
|
|
void kvm_arm_halt_vcpu(struct kvm_vcpu *vcpu);
|
|
|
|
void kvm_arm_resume_vcpu(struct kvm_vcpu *vcpu);
|
2012-12-11 00:29:28 +08:00
|
|
|
|
2016-02-16 20:52:39 +08:00
|
|
|
u64 __kvm_call_hyp(void *hypfn, ...);
|
2016-03-01 21:12:44 +08:00
|
|
|
#define kvm_call_hyp(f, ...) __kvm_call_hyp(kvm_ksym_ref(f), ##__VA_ARGS__)
|
|
|
|
|
2014-10-16 23:00:18 +08:00
|
|
|
void force_vm_exit(const cpumask_t *mask);
|
2015-01-16 07:58:59 +08:00
|
|
|
void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
|
2012-12-11 00:29:28 +08:00
|
|
|
|
|
|
|
int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
|
|
|
|
int exception_index);
|
|
|
|
|
|
|
|
int kvm_perf_init(void);
|
|
|
|
int kvm_perf_teardown(void);
|
|
|
|
|
2014-06-02 21:37:13 +08:00
|
|
|
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
|
|
|
|
|
2016-07-01 01:40:45 +08:00
|
|
|
static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
|
2012-12-18 01:07:52 +08:00
|
|
|
unsigned long hyp_stack_ptr,
|
|
|
|
unsigned long vector_ptr)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Call initialization code, and switch to the full blown
|
|
|
|
* HYP code.
|
|
|
|
*/
|
2016-07-01 01:40:44 +08:00
|
|
|
__kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr);
|
2012-12-18 01:07:52 +08:00
|
|
|
}
|
|
|
|
|
2016-07-01 01:40:44 +08:00
|
|
|
void __kvm_hyp_teardown(void);
|
2016-07-01 01:40:48 +08:00
|
|
|
static inline void __cpu_reset_hyp_mode(unsigned long vector_ptr,
|
|
|
|
phys_addr_t phys_idmap_start)
|
arm64: kvm: allows kvm cpu hotplug
The current kvm implementation on arm64 does cpu-specific initialization
at system boot, and has no way to gracefully shutdown a core in terms of
kvm. This prevents kexec from rebooting the system at EL2.
This patch adds a cpu tear-down function and also puts an existing cpu-init
code into a separate function, kvm_arch_hardware_disable() and
kvm_arch_hardware_enable() respectively.
We don't need the arm64 specific cpu hotplug hook any more.
Since this patch modifies common code between arm and arm64, one stub
definition, __cpu_reset_hyp_mode(), is added on arm side to avoid
compilation errors.
Signed-off-by: AKASHI Takahiro <takahiro.akashi@linaro.org>
[Rebase, added separate VHE init/exit path, changed resets use of
kvm_call_hyp() to the __version, en/disabled hardware in init_subsystems(),
added icache maintenance to __kvm_hyp_reset() and removed lr restore, removed
guest-enter after teardown handling]
Signed-off-by: James Morse <james.morse@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2016-04-28 00:47:05 +08:00
|
|
|
{
|
2016-07-01 01:40:44 +08:00
|
|
|
kvm_call_hyp(__kvm_hyp_teardown, phys_idmap_start);
|
arm64: kvm: allows kvm cpu hotplug
The current kvm implementation on arm64 does cpu-specific initialization
at system boot, and has no way to gracefully shutdown a core in terms of
kvm. This prevents kexec from rebooting the system at EL2.
This patch adds a cpu tear-down function and also puts an existing cpu-init
code into a separate function, kvm_arch_hardware_disable() and
kvm_arch_hardware_enable() respectively.
We don't need the arm64 specific cpu hotplug hook any more.
Since this patch modifies common code between arm and arm64, one stub
definition, __cpu_reset_hyp_mode(), is added on arm side to avoid
compilation errors.
Signed-off-by: AKASHI Takahiro <takahiro.akashi@linaro.org>
[Rebase, added separate VHE init/exit path, changed resets use of
kvm_call_hyp() to the __version, en/disabled hardware in init_subsystems(),
added icache maintenance to __kvm_hyp_reset() and removed lr restore, removed
guest-enter after teardown handling]
Signed-off-by: James Morse <james.morse@arm.com>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2016-04-28 00:47:05 +08:00
|
|
|
}
|
|
|
|
|
2014-08-28 21:13:02 +08:00
|
|
|
static inline void kvm_arch_hardware_unsetup(void) {}
|
|
|
|
static inline void kvm_arch_sync_events(struct kvm *kvm) {}
|
|
|
|
static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
|
|
|
|
static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
|
2016-05-13 18:16:35 +08:00
|
|
|
static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
|
2014-08-28 21:13:02 +08:00
|
|
|
|
2015-07-08 00:29:56 +08:00
|
|
|
void kvm_arm_init_debug(void);
|
|
|
|
void kvm_arm_setup_debug(struct kvm_vcpu *vcpu);
|
|
|
|
void kvm_arm_clear_debug(struct kvm_vcpu *vcpu);
|
2015-07-08 00:30:00 +08:00
|
|
|
void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu);
|
2016-01-11 21:35:32 +08:00
|
|
|
int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
|
|
|
|
struct kvm_device_attr *attr);
|
|
|
|
int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
|
|
|
|
struct kvm_device_attr *attr);
|
|
|
|
int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
|
|
|
|
struct kvm_device_attr *attr);
|
2015-07-08 00:29:56 +08:00
|
|
|
|
2016-02-22 18:57:30 +08:00
|
|
|
static inline void __cpu_init_stage2(void)
|
|
|
|
{
|
2016-04-05 23:11:47 +08:00
|
|
|
u32 parange = kvm_call_hyp(__init_stage2_translation);
|
|
|
|
|
|
|
|
WARN_ONCE(parange < 40,
|
|
|
|
"PARange is %d bits, unsupported configuration!", parange);
|
2016-02-22 18:57:30 +08:00
|
|
|
}
|
|
|
|
|
2012-12-11 00:29:28 +08:00
|
|
|
#endif /* __ARM64_KVM_HOST_H__ */
|