mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-04 09:34:12 +08:00
cdeb5d7d89
We call idle_kvm_start_guest() from power7_offline() if the thread has been requested to enter KVM. We pass it the SRR1 value that was returned from power7_idle_insn() which tells us what sort of wakeup we're processing. Depending on the SRR1 value we pass in, the KVM code might enter the guest, or it might return to us to do some host action if the wakeup requires it. If idle_kvm_start_guest() is able to handle the wakeup, and enter the guest it is supposed to indicate that by returning a zero SRR1 value to us. That was the behaviour prior to commit10d91611f4("powerpc/64s: Reimplement book3s idle code in C"), however in that commit the handling of SRR1 was reworked, and the zeroing behaviour was lost. Returning from idle_kvm_start_guest() without zeroing the SRR1 value can confuse the host offline code, causing the guest to crash and other weirdness. Fixes:10d91611f4("powerpc/64s: Reimplement book3s idle code in C") Cc: stable@vger.kernel.org # v5.2+ Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://lore.kernel.org/r/20211015133929.832061-2-mpe@ellerman.id.au
3062 lines
73 KiB
ArmAsm
3062 lines
73 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
*
|
|
* Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
|
|
*
|
|
* Derived from book3s_rmhandlers.S and other files, which are:
|
|
*
|
|
* Copyright SUSE Linux Products GmbH 2009
|
|
*
|
|
* Authors: Alexander Graf <agraf@suse.de>
|
|
*/
|
|
|
|
#include <asm/ppc_asm.h>
|
|
#include <asm/code-patching-asm.h>
|
|
#include <asm/kvm_asm.h>
|
|
#include <asm/reg.h>
|
|
#include <asm/mmu.h>
|
|
#include <asm/page.h>
|
|
#include <asm/ptrace.h>
|
|
#include <asm/hvcall.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/exception-64s.h>
|
|
#include <asm/kvm_book3s_asm.h>
|
|
#include <asm/book3s/64/mmu-hash.h>
|
|
#include <asm/export.h>
|
|
#include <asm/tm.h>
|
|
#include <asm/opal.h>
|
|
#include <asm/thread_info.h>
|
|
#include <asm/asm-compat.h>
|
|
#include <asm/feature-fixups.h>
|
|
#include <asm/cpuidle.h>
|
|
|
|
/* Values in HSTATE_NAPPING(r13) */
|
|
#define NAPPING_CEDE 1
|
|
#define NAPPING_NOVCPU 2
|
|
#define NAPPING_UNSPLIT 3
|
|
|
|
/* Stack frame offsets for kvmppc_hv_entry */
|
|
#define SFS 160
|
|
#define STACK_SLOT_TRAP (SFS-4)
|
|
#define STACK_SLOT_TID (SFS-16)
|
|
#define STACK_SLOT_PSSCR (SFS-24)
|
|
#define STACK_SLOT_PID (SFS-32)
|
|
#define STACK_SLOT_IAMR (SFS-40)
|
|
#define STACK_SLOT_CIABR (SFS-48)
|
|
#define STACK_SLOT_DAWR0 (SFS-56)
|
|
#define STACK_SLOT_DAWRX0 (SFS-64)
|
|
#define STACK_SLOT_HFSCR (SFS-72)
|
|
#define STACK_SLOT_AMR (SFS-80)
|
|
#define STACK_SLOT_UAMOR (SFS-88)
|
|
#define STACK_SLOT_FSCR (SFS-96)
|
|
|
|
/*
|
|
* Call kvmppc_hv_entry in real mode.
|
|
* Must be called with interrupts hard-disabled.
|
|
*
|
|
* Input Registers:
|
|
*
|
|
* LR = return address to continue at after eventually re-enabling MMU
|
|
*/
|
|
_GLOBAL_TOC(kvmppc_hv_entry_trampoline)
|
|
mflr r0
|
|
std r0, PPC_LR_STKOFF(r1)
|
|
stdu r1, -112(r1)
|
|
mfmsr r10
|
|
std r10, HSTATE_HOST_MSR(r13)
|
|
LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
|
|
li r0,MSR_RI
|
|
andc r0,r10,r0
|
|
li r6,MSR_IR | MSR_DR
|
|
andc r6,r10,r6
|
|
mtmsrd r0,1 /* clear RI in MSR */
|
|
mtsrr0 r5
|
|
mtsrr1 r6
|
|
RFI_TO_KERNEL
|
|
|
|
kvmppc_call_hv_entry:
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
bl kvmppc_hv_entry
|
|
|
|
/* Back from guest - restore host state and return to caller */
|
|
|
|
BEGIN_FTR_SECTION
|
|
/* Restore host DABR and DABRX */
|
|
ld r5,HSTATE_DABR(r13)
|
|
li r6,7
|
|
mtspr SPRN_DABR,r5
|
|
mtspr SPRN_DABRX,r6
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
|
|
|
|
/* Restore SPRG3 */
|
|
ld r3,PACA_SPRG_VDSO(r13)
|
|
mtspr SPRN_SPRG_VDSO_WRITE,r3
|
|
|
|
/* Reload the host's PMU registers */
|
|
bl kvmhv_load_host_pmu
|
|
|
|
/*
|
|
* Reload DEC. HDEC interrupts were disabled when
|
|
* we reloaded the host's LPCR value.
|
|
*/
|
|
ld r3, HSTATE_DECEXP(r13)
|
|
mftb r4
|
|
subf r4, r4, r3
|
|
mtspr SPRN_DEC, r4
|
|
|
|
/* hwthread_req may have got set by cede or no vcpu, so clear it */
|
|
li r0, 0
|
|
stb r0, HSTATE_HWTHREAD_REQ(r13)
|
|
|
|
/*
|
|
* For external interrupts we need to call the Linux
|
|
* handler to process the interrupt. We do that by jumping
|
|
* to absolute address 0x500 for external interrupts.
|
|
* The [h]rfid at the end of the handler will return to
|
|
* the book3s_hv_interrupts.S code. For other interrupts
|
|
* we do the rfid to get back to the book3s_hv_interrupts.S
|
|
* code here.
|
|
*/
|
|
ld r8, 112+PPC_LR_STKOFF(r1)
|
|
addi r1, r1, 112
|
|
ld r7, HSTATE_HOST_MSR(r13)
|
|
|
|
/* Return the trap number on this thread as the return value */
|
|
mr r3, r12
|
|
|
|
/* RFI into the highmem handler */
|
|
mfmsr r6
|
|
li r0, MSR_RI
|
|
andc r6, r6, r0
|
|
mtmsrd r6, 1 /* Clear RI in MSR */
|
|
mtsrr0 r8
|
|
mtsrr1 r7
|
|
RFI_TO_KERNEL
|
|
|
|
kvmppc_primary_no_guest:
|
|
/* We handle this much like a ceded vcpu */
|
|
/* put the HDEC into the DEC, since HDEC interrupts don't wake us */
|
|
/* HDEC may be larger than DEC for arch >= v3.00, but since the */
|
|
/* HDEC value came from DEC in the first place, it will fit */
|
|
mfspr r3, SPRN_HDEC
|
|
mtspr SPRN_DEC, r3
|
|
/*
|
|
* Make sure the primary has finished the MMU switch.
|
|
* We should never get here on a secondary thread, but
|
|
* check it for robustness' sake.
|
|
*/
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
65: lbz r0, VCORE_IN_GUEST(r5)
|
|
cmpwi r0, 0
|
|
beq 65b
|
|
/* Set LPCR. */
|
|
ld r8,VCORE_LPCR(r5)
|
|
mtspr SPRN_LPCR,r8
|
|
isync
|
|
/* set our bit in napping_threads */
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
lbz r7, HSTATE_PTID(r13)
|
|
li r0, 1
|
|
sld r0, r0, r7
|
|
addi r6, r5, VCORE_NAPPING_THREADS
|
|
1: lwarx r3, 0, r6
|
|
or r3, r3, r0
|
|
stwcx. r3, 0, r6
|
|
bne 1b
|
|
/* order napping_threads update vs testing entry_exit_map */
|
|
isync
|
|
li r12, 0
|
|
lwz r7, VCORE_ENTRY_EXIT(r5)
|
|
cmpwi r7, 0x100
|
|
bge kvm_novcpu_exit /* another thread already exiting */
|
|
li r3, NAPPING_NOVCPU
|
|
stb r3, HSTATE_NAPPING(r13)
|
|
|
|
li r3, 0 /* Don't wake on privileged (OS) doorbell */
|
|
b kvm_do_nap
|
|
|
|
/*
|
|
* kvm_novcpu_wakeup
|
|
* Entered from kvm_start_guest if kvm_hstate.napping is set
|
|
* to NAPPING_NOVCPU
|
|
* r2 = kernel TOC
|
|
* r13 = paca
|
|
*/
|
|
kvm_novcpu_wakeup:
|
|
ld r1, HSTATE_HOST_R1(r13)
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
li r0, 0
|
|
stb r0, HSTATE_NAPPING(r13)
|
|
|
|
/* check the wake reason */
|
|
bl kvmppc_check_wake_reason
|
|
|
|
/*
|
|
* Restore volatile registers since we could have called
|
|
* a C routine in kvmppc_check_wake_reason.
|
|
* r5 = VCORE
|
|
*/
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
|
|
/* see if any other thread is already exiting */
|
|
lwz r0, VCORE_ENTRY_EXIT(r5)
|
|
cmpwi r0, 0x100
|
|
bge kvm_novcpu_exit
|
|
|
|
/* clear our bit in napping_threads */
|
|
lbz r7, HSTATE_PTID(r13)
|
|
li r0, 1
|
|
sld r0, r0, r7
|
|
addi r6, r5, VCORE_NAPPING_THREADS
|
|
4: lwarx r7, 0, r6
|
|
andc r7, r7, r0
|
|
stwcx. r7, 0, r6
|
|
bne 4b
|
|
|
|
/* See if the wake reason means we need to exit */
|
|
cmpdi r3, 0
|
|
bge kvm_novcpu_exit
|
|
|
|
/* See if our timeslice has expired (HDEC is negative) */
|
|
mfspr r0, SPRN_HDEC
|
|
extsw r0, r0
|
|
li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
|
|
cmpdi r0, 0
|
|
blt kvm_novcpu_exit
|
|
|
|
/* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
cmpdi r4, 0
|
|
beq kvmppc_primary_no_guest
|
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
addi r3, r4, VCPU_TB_RMENTRY
|
|
bl kvmhv_start_timing
|
|
#endif
|
|
b kvmppc_got_guest
|
|
|
|
kvm_novcpu_exit:
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
cmpdi r4, 0
|
|
beq 13f
|
|
addi r3, r4, VCPU_TB_RMEXIT
|
|
bl kvmhv_accumulate_time
|
|
#endif
|
|
13: mr r3, r12
|
|
stw r12, STACK_SLOT_TRAP(r1)
|
|
bl kvmhv_commence_exit
|
|
nop
|
|
b kvmhv_switch_to_host
|
|
|
|
/*
|
|
* We come in here when wakened from Linux offline idle code.
|
|
* Relocation is off
|
|
* r3 contains the SRR1 wakeup value, SRR1 is trashed.
|
|
*/
|
|
_GLOBAL(idle_kvm_start_guest)
|
|
mfcr r5
|
|
mflr r0
|
|
std r5, 8(r1) // Save CR in caller's frame
|
|
std r0, 16(r1) // Save LR in caller's frame
|
|
// Create frame on emergency stack
|
|
ld r4, PACAEMERGSP(r13)
|
|
stdu r1, -SWITCH_FRAME_SIZE(r4)
|
|
// Switch to new frame on emergency stack
|
|
mr r1, r4
|
|
std r3, 32(r1) // Save SRR1 wakeup value
|
|
SAVE_NVGPRS(r1)
|
|
|
|
/*
|
|
* Could avoid this and pass it through in r3. For now,
|
|
* code expects it to be in SRR1.
|
|
*/
|
|
mtspr SPRN_SRR1,r3
|
|
|
|
li r0,0
|
|
stb r0,PACA_FTRACE_ENABLED(r13)
|
|
|
|
li r0,KVM_HWTHREAD_IN_KVM
|
|
stb r0,HSTATE_HWTHREAD_STATE(r13)
|
|
|
|
/* kvm cede / napping does not come through here */
|
|
lbz r0,HSTATE_NAPPING(r13)
|
|
twnei r0,0
|
|
|
|
b 1f
|
|
|
|
kvm_unsplit_wakeup:
|
|
li r0, 0
|
|
stb r0, HSTATE_NAPPING(r13)
|
|
|
|
1:
|
|
|
|
/*
|
|
* We weren't napping due to cede, so this must be a secondary
|
|
* thread being woken up to run a guest, or being woken up due
|
|
* to a stray IPI. (Or due to some machine check or hypervisor
|
|
* maintenance interrupt while the core is in KVM.)
|
|
*/
|
|
|
|
/* Check the wake reason in SRR1 to see why we got here */
|
|
bl kvmppc_check_wake_reason
|
|
/*
|
|
* kvmppc_check_wake_reason could invoke a C routine, but we
|
|
* have no volatile registers to restore when we return.
|
|
*/
|
|
|
|
cmpdi r3, 0
|
|
bge kvm_no_guest
|
|
|
|
/* get vcore pointer, NULL if we have nothing to run */
|
|
ld r5,HSTATE_KVM_VCORE(r13)
|
|
cmpdi r5,0
|
|
/* if we have no vcore to run, go back to sleep */
|
|
beq kvm_no_guest
|
|
|
|
kvm_secondary_got_guest:
|
|
|
|
// About to go to guest, clear saved SRR1
|
|
li r0, 0
|
|
std r0, 32(r1)
|
|
|
|
/* Set HSTATE_DSCR(r13) to something sensible */
|
|
ld r6, PACA_DSCR_DEFAULT(r13)
|
|
std r6, HSTATE_DSCR(r13)
|
|
|
|
/* On thread 0 of a subcore, set HDEC to max */
|
|
lbz r4, HSTATE_PTID(r13)
|
|
cmpwi r4, 0
|
|
bne 63f
|
|
lis r6,0x7fff /* MAX_INT@h */
|
|
mtspr SPRN_HDEC, r6
|
|
/* and set per-LPAR registers, if doing dynamic micro-threading */
|
|
ld r6, HSTATE_SPLIT_MODE(r13)
|
|
cmpdi r6, 0
|
|
beq 63f
|
|
ld r0, KVM_SPLIT_RPR(r6)
|
|
mtspr SPRN_RPR, r0
|
|
ld r0, KVM_SPLIT_PMMAR(r6)
|
|
mtspr SPRN_PMMAR, r0
|
|
ld r0, KVM_SPLIT_LDBAR(r6)
|
|
mtspr SPRN_LDBAR, r0
|
|
isync
|
|
63:
|
|
/* Order load of vcpu after load of vcore */
|
|
lwsync
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
bl kvmppc_hv_entry
|
|
|
|
/* Back from the guest, go back to nap */
|
|
/* Clear our vcpu and vcore pointers so we don't come back in early */
|
|
li r0, 0
|
|
std r0, HSTATE_KVM_VCPU(r13)
|
|
/*
|
|
* Once we clear HSTATE_KVM_VCORE(r13), the code in
|
|
* kvmppc_run_core() is going to assume that all our vcpu
|
|
* state is visible in memory. This lwsync makes sure
|
|
* that that is true.
|
|
*/
|
|
lwsync
|
|
std r0, HSTATE_KVM_VCORE(r13)
|
|
|
|
/*
|
|
* All secondaries exiting guest will fall through this path.
|
|
* Before proceeding, just check for HMI interrupt and
|
|
* invoke opal hmi handler. By now we are sure that the
|
|
* primary thread on this core/subcore has already made partition
|
|
* switch/TB resync and we are good to call opal hmi handler.
|
|
*/
|
|
cmpwi r12, BOOK3S_INTERRUPT_HMI
|
|
bne kvm_no_guest
|
|
|
|
li r3,0 /* NULL argument */
|
|
bl hmi_exception_realmode
|
|
/*
|
|
* At this point we have finished executing in the guest.
|
|
* We need to wait for hwthread_req to become zero, since
|
|
* we may not turn on the MMU while hwthread_req is non-zero.
|
|
* While waiting we also need to check if we get given a vcpu to run.
|
|
*/
|
|
kvm_no_guest:
|
|
lbz r3, HSTATE_HWTHREAD_REQ(r13)
|
|
cmpwi r3, 0
|
|
bne 53f
|
|
HMT_MEDIUM
|
|
li r0, KVM_HWTHREAD_IN_KERNEL
|
|
stb r0, HSTATE_HWTHREAD_STATE(r13)
|
|
/* need to recheck hwthread_req after a barrier, to avoid race */
|
|
sync
|
|
lbz r3, HSTATE_HWTHREAD_REQ(r13)
|
|
cmpwi r3, 0
|
|
bne 54f
|
|
|
|
/*
|
|
* Jump to idle_return_gpr_loss, which returns to the
|
|
* idle_kvm_start_guest caller.
|
|
*/
|
|
li r3, LPCR_PECE0
|
|
mfspr r4, SPRN_LPCR
|
|
rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
|
|
mtspr SPRN_LPCR, r4
|
|
// Return SRR1 wakeup value, or 0 if we went into the guest
|
|
ld r3, 32(r1)
|
|
REST_NVGPRS(r1)
|
|
ld r1, 0(r1) // Switch back to caller stack
|
|
ld r0, 16(r1) // Reload LR
|
|
ld r5, 8(r1) // Reload CR
|
|
mtlr r0
|
|
mtcr r5
|
|
blr
|
|
|
|
53:
|
|
HMT_LOW
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
cmpdi r5, 0
|
|
bne 60f
|
|
ld r3, HSTATE_SPLIT_MODE(r13)
|
|
cmpdi r3, 0
|
|
beq kvm_no_guest
|
|
lbz r0, KVM_SPLIT_DO_NAP(r3)
|
|
cmpwi r0, 0
|
|
beq kvm_no_guest
|
|
HMT_MEDIUM
|
|
b kvm_unsplit_nap
|
|
60: HMT_MEDIUM
|
|
b kvm_secondary_got_guest
|
|
|
|
54: li r0, KVM_HWTHREAD_IN_KVM
|
|
stb r0, HSTATE_HWTHREAD_STATE(r13)
|
|
b kvm_no_guest
|
|
|
|
/*
|
|
* Here the primary thread is trying to return the core to
|
|
* whole-core mode, so we need to nap.
|
|
*/
|
|
kvm_unsplit_nap:
|
|
/*
|
|
* When secondaries are napping in kvm_unsplit_nap() with
|
|
* hwthread_req = 1, HMI goes ignored even though subcores are
|
|
* already exited the guest. Hence HMI keeps waking up secondaries
|
|
* from nap in a loop and secondaries always go back to nap since
|
|
* no vcore is assigned to them. This makes impossible for primary
|
|
* thread to get hold of secondary threads resulting into a soft
|
|
* lockup in KVM path.
|
|
*
|
|
* Let us check if HMI is pending and handle it before we go to nap.
|
|
*/
|
|
cmpwi r12, BOOK3S_INTERRUPT_HMI
|
|
bne 55f
|
|
li r3, 0 /* NULL argument */
|
|
bl hmi_exception_realmode
|
|
55:
|
|
/*
|
|
* Ensure that secondary doesn't nap when it has
|
|
* its vcore pointer set.
|
|
*/
|
|
sync /* matches smp_mb() before setting split_info.do_nap */
|
|
ld r0, HSTATE_KVM_VCORE(r13)
|
|
cmpdi r0, 0
|
|
bne kvm_no_guest
|
|
/* clear any pending message */
|
|
BEGIN_FTR_SECTION
|
|
lis r6, (PPC_DBELL_SERVER << (63-36))@h
|
|
PPC_MSGCLR(6)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
/* Set kvm_split_mode.napped[tid] = 1 */
|
|
ld r3, HSTATE_SPLIT_MODE(r13)
|
|
li r0, 1
|
|
lhz r4, PACAPACAINDEX(r13)
|
|
clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */
|
|
addi r4, r4, KVM_SPLIT_NAPPED
|
|
stbx r0, r3, r4
|
|
/* Check the do_nap flag again after setting napped[] */
|
|
sync
|
|
lbz r0, KVM_SPLIT_DO_NAP(r3)
|
|
cmpwi r0, 0
|
|
beq 57f
|
|
li r3, NAPPING_UNSPLIT
|
|
stb r3, HSTATE_NAPPING(r13)
|
|
li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
|
|
mfspr r5, SPRN_LPCR
|
|
rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
|
|
b kvm_nap_sequence
|
|
|
|
57: li r0, 0
|
|
stbx r0, r3, r4
|
|
b kvm_no_guest
|
|
|
|
/******************************************************************************
|
|
* *
|
|
* Entry code *
|
|
* *
|
|
*****************************************************************************/
|
|
|
|
.global kvmppc_hv_entry
|
|
kvmppc_hv_entry:
|
|
|
|
/* Required state:
|
|
*
|
|
* R4 = vcpu pointer (or NULL)
|
|
* MSR = ~IR|DR
|
|
* R13 = PACA
|
|
* R1 = host R1
|
|
* R2 = TOC
|
|
* all other volatile GPRS = free
|
|
* Does not preserve non-volatile GPRs or CR fields
|
|
*/
|
|
mflr r0
|
|
std r0, PPC_LR_STKOFF(r1)
|
|
stdu r1, -SFS(r1)
|
|
|
|
/* Save R1 in the PACA */
|
|
std r1, HSTATE_HOST_R1(r13)
|
|
|
|
li r6, KVM_GUEST_MODE_HOST_HV
|
|
stb r6, HSTATE_IN_GUEST(r13)
|
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
/* Store initial timestamp */
|
|
cmpdi r4, 0
|
|
beq 1f
|
|
addi r3, r4, VCPU_TB_RMENTRY
|
|
bl kvmhv_start_timing
|
|
1:
|
|
#endif
|
|
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
ld r9, VCORE_KVM(r5) /* pointer to struct kvm */
|
|
|
|
/*
|
|
* POWER7/POWER8 host -> guest partition switch code.
|
|
* We don't have to lock against concurrent tlbies,
|
|
* but we do have to coordinate across hardware threads.
|
|
*/
|
|
/* Set bit in entry map iff exit map is zero. */
|
|
li r7, 1
|
|
lbz r6, HSTATE_PTID(r13)
|
|
sld r7, r7, r6
|
|
addi r8, r5, VCORE_ENTRY_EXIT
|
|
21: lwarx r3, 0, r8
|
|
cmpwi r3, 0x100 /* any threads starting to exit? */
|
|
bge secondary_too_late /* if so we're too late to the party */
|
|
or r3, r3, r7
|
|
stwcx. r3, 0, r8
|
|
bne 21b
|
|
|
|
/* Primary thread switches to guest partition. */
|
|
cmpwi r6,0
|
|
bne 10f
|
|
|
|
lwz r7,KVM_LPID(r9)
|
|
ld r6,KVM_SDR1(r9)
|
|
li r0,LPID_RSVD /* switch to reserved LPID */
|
|
mtspr SPRN_LPID,r0
|
|
ptesync
|
|
mtspr SPRN_SDR1,r6 /* switch to partition page table */
|
|
mtspr SPRN_LPID,r7
|
|
isync
|
|
|
|
/* See if we need to flush the TLB. */
|
|
mr r3, r9 /* kvm pointer */
|
|
lhz r4, PACAPACAINDEX(r13) /* physical cpu number */
|
|
li r5, 0 /* nested vcpu pointer */
|
|
bl kvmppc_check_need_tlb_flush
|
|
nop
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
|
|
/* Add timebase offset onto timebase */
|
|
22: ld r8,VCORE_TB_OFFSET(r5)
|
|
cmpdi r8,0
|
|
beq 37f
|
|
std r8, VCORE_TB_OFFSET_APPL(r5)
|
|
mftb r6 /* current host timebase */
|
|
add r8,r8,r6
|
|
mtspr SPRN_TBU40,r8 /* update upper 40 bits */
|
|
mftb r7 /* check if lower 24 bits overflowed */
|
|
clrldi r6,r6,40
|
|
clrldi r7,r7,40
|
|
cmpld r7,r6
|
|
bge 37f
|
|
addis r8,r8,0x100 /* if so, increment upper 40 bits */
|
|
mtspr SPRN_TBU40,r8
|
|
|
|
/* Load guest PCR value to select appropriate compat mode */
|
|
37: ld r7, VCORE_PCR(r5)
|
|
LOAD_REG_IMMEDIATE(r6, PCR_MASK)
|
|
cmpld r7, r6
|
|
beq 38f
|
|
or r7, r7, r6
|
|
mtspr SPRN_PCR, r7
|
|
38:
|
|
|
|
BEGIN_FTR_SECTION
|
|
/* DPDES and VTB are shared between threads */
|
|
ld r8, VCORE_DPDES(r5)
|
|
ld r7, VCORE_VTB(r5)
|
|
mtspr SPRN_DPDES, r8
|
|
mtspr SPRN_VTB, r7
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
|
|
/* Mark the subcore state as inside guest */
|
|
bl kvmppc_subcore_enter_guest
|
|
nop
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
li r0,1
|
|
stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
|
|
|
|
/* Do we have a guest vcpu to run? */
|
|
10: cmpdi r4, 0
|
|
beq kvmppc_primary_no_guest
|
|
kvmppc_got_guest:
|
|
/* Increment yield count if they have a VPA */
|
|
ld r3, VCPU_VPA(r4)
|
|
cmpdi r3, 0
|
|
beq 25f
|
|
li r6, LPPACA_YIELDCOUNT
|
|
LWZX_BE r5, r3, r6
|
|
addi r5, r5, 1
|
|
STWX_BE r5, r3, r6
|
|
li r6, 1
|
|
stb r6, VCPU_VPA_DIRTY(r4)
|
|
25:
|
|
|
|
/* Save purr/spurr */
|
|
mfspr r5,SPRN_PURR
|
|
mfspr r6,SPRN_SPURR
|
|
std r5,HSTATE_PURR(r13)
|
|
std r6,HSTATE_SPURR(r13)
|
|
ld r7,VCPU_PURR(r4)
|
|
ld r8,VCPU_SPURR(r4)
|
|
mtspr SPRN_PURR,r7
|
|
mtspr SPRN_SPURR,r8
|
|
|
|
/* Save host values of some registers */
|
|
BEGIN_FTR_SECTION
|
|
mfspr r5, SPRN_CIABR
|
|
mfspr r6, SPRN_DAWR0
|
|
mfspr r7, SPRN_DAWRX0
|
|
mfspr r8, SPRN_IAMR
|
|
std r5, STACK_SLOT_CIABR(r1)
|
|
std r6, STACK_SLOT_DAWR0(r1)
|
|
std r7, STACK_SLOT_DAWRX0(r1)
|
|
std r8, STACK_SLOT_IAMR(r1)
|
|
mfspr r5, SPRN_FSCR
|
|
std r5, STACK_SLOT_FSCR(r1)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
|
|
mfspr r5, SPRN_AMR
|
|
std r5, STACK_SLOT_AMR(r1)
|
|
mfspr r6, SPRN_UAMOR
|
|
std r6, STACK_SLOT_UAMOR(r1)
|
|
|
|
BEGIN_FTR_SECTION
|
|
/* Set partition DABR */
|
|
/* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
|
|
lwz r5,VCPU_DABRX(r4)
|
|
ld r6,VCPU_DABR(r4)
|
|
mtspr SPRN_DABRX,r5
|
|
mtspr SPRN_DABR,r6
|
|
isync
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
|
|
|
|
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
|
|
BEGIN_FTR_SECTION
|
|
b 91f
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_TM)
|
|
/*
|
|
* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
|
|
*/
|
|
mr r3, r4
|
|
ld r4, VCPU_MSR(r3)
|
|
li r5, 0 /* don't preserve non-vol regs */
|
|
bl kvmppc_restore_tm_hv
|
|
nop
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
91:
|
|
#endif
|
|
|
|
/* Load guest PMU registers; r4 = vcpu pointer here */
|
|
mr r3, r4
|
|
bl kvmhv_load_guest_pmu
|
|
|
|
/* Load up FP, VMX and VSX registers */
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
bl kvmppc_load_fp
|
|
|
|
ld r14, VCPU_GPR(R14)(r4)
|
|
ld r15, VCPU_GPR(R15)(r4)
|
|
ld r16, VCPU_GPR(R16)(r4)
|
|
ld r17, VCPU_GPR(R17)(r4)
|
|
ld r18, VCPU_GPR(R18)(r4)
|
|
ld r19, VCPU_GPR(R19)(r4)
|
|
ld r20, VCPU_GPR(R20)(r4)
|
|
ld r21, VCPU_GPR(R21)(r4)
|
|
ld r22, VCPU_GPR(R22)(r4)
|
|
ld r23, VCPU_GPR(R23)(r4)
|
|
ld r24, VCPU_GPR(R24)(r4)
|
|
ld r25, VCPU_GPR(R25)(r4)
|
|
ld r26, VCPU_GPR(R26)(r4)
|
|
ld r27, VCPU_GPR(R27)(r4)
|
|
ld r28, VCPU_GPR(R28)(r4)
|
|
ld r29, VCPU_GPR(R29)(r4)
|
|
ld r30, VCPU_GPR(R30)(r4)
|
|
ld r31, VCPU_GPR(R31)(r4)
|
|
|
|
/* Switch DSCR to guest value */
|
|
ld r5, VCPU_DSCR(r4)
|
|
mtspr SPRN_DSCR, r5
|
|
|
|
BEGIN_FTR_SECTION
|
|
/* Skip next section on POWER7 */
|
|
b 8f
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
|
|
/* Load up POWER8-specific registers */
|
|
ld r5, VCPU_IAMR(r4)
|
|
lwz r6, VCPU_PSPB(r4)
|
|
ld r7, VCPU_FSCR(r4)
|
|
mtspr SPRN_IAMR, r5
|
|
mtspr SPRN_PSPB, r6
|
|
mtspr SPRN_FSCR, r7
|
|
/*
|
|
* Handle broken DAWR case by not writing it. This means we
|
|
* can still store the DAWR register for migration.
|
|
*/
|
|
LOAD_REG_ADDR(r5, dawr_force_enable)
|
|
lbz r5, 0(r5)
|
|
cmpdi r5, 0
|
|
beq 1f
|
|
ld r5, VCPU_DAWR0(r4)
|
|
ld r6, VCPU_DAWRX0(r4)
|
|
mtspr SPRN_DAWR0, r5
|
|
mtspr SPRN_DAWRX0, r6
|
|
1:
|
|
ld r7, VCPU_CIABR(r4)
|
|
ld r8, VCPU_TAR(r4)
|
|
mtspr SPRN_CIABR, r7
|
|
mtspr SPRN_TAR, r8
|
|
ld r5, VCPU_IC(r4)
|
|
ld r8, VCPU_EBBHR(r4)
|
|
mtspr SPRN_IC, r5
|
|
mtspr SPRN_EBBHR, r8
|
|
ld r5, VCPU_EBBRR(r4)
|
|
ld r6, VCPU_BESCR(r4)
|
|
lwz r7, VCPU_GUEST_PID(r4)
|
|
ld r8, VCPU_WORT(r4)
|
|
mtspr SPRN_EBBRR, r5
|
|
mtspr SPRN_BESCR, r6
|
|
mtspr SPRN_PID, r7
|
|
mtspr SPRN_WORT, r8
|
|
/* POWER8-only registers */
|
|
ld r5, VCPU_TCSCR(r4)
|
|
ld r6, VCPU_ACOP(r4)
|
|
ld r7, VCPU_CSIGR(r4)
|
|
ld r8, VCPU_TACR(r4)
|
|
mtspr SPRN_TCSCR, r5
|
|
mtspr SPRN_ACOP, r6
|
|
mtspr SPRN_CSIGR, r7
|
|
mtspr SPRN_TACR, r8
|
|
nop
|
|
8:
|
|
|
|
ld r5, VCPU_SPRG0(r4)
|
|
ld r6, VCPU_SPRG1(r4)
|
|
ld r7, VCPU_SPRG2(r4)
|
|
ld r8, VCPU_SPRG3(r4)
|
|
mtspr SPRN_SPRG0, r5
|
|
mtspr SPRN_SPRG1, r6
|
|
mtspr SPRN_SPRG2, r7
|
|
mtspr SPRN_SPRG3, r8
|
|
|
|
/* Load up DAR and DSISR */
|
|
ld r5, VCPU_DAR(r4)
|
|
lwz r6, VCPU_DSISR(r4)
|
|
mtspr SPRN_DAR, r5
|
|
mtspr SPRN_DSISR, r6
|
|
|
|
/* Restore AMR and UAMOR, set AMOR to all 1s */
|
|
ld r5,VCPU_AMR(r4)
|
|
ld r6,VCPU_UAMOR(r4)
|
|
li r7,-1
|
|
mtspr SPRN_AMR,r5
|
|
mtspr SPRN_UAMOR,r6
|
|
mtspr SPRN_AMOR,r7
|
|
|
|
/* Restore state of CTRL run bit; assume 1 on entry */
|
|
lwz r5,VCPU_CTRL(r4)
|
|
andi. r5,r5,1
|
|
bne 4f
|
|
mfspr r6,SPRN_CTRLF
|
|
clrrdi r6,r6,1
|
|
mtspr SPRN_CTRLT,r6
|
|
4:
|
|
/* Secondary threads wait for primary to have done partition switch */
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
lbz r6, HSTATE_PTID(r13)
|
|
cmpwi r6, 0
|
|
beq 21f
|
|
lbz r0, VCORE_IN_GUEST(r5)
|
|
cmpwi r0, 0
|
|
bne 21f
|
|
HMT_LOW
|
|
20: lwz r3, VCORE_ENTRY_EXIT(r5)
|
|
cmpwi r3, 0x100
|
|
bge no_switch_exit
|
|
lbz r0, VCORE_IN_GUEST(r5)
|
|
cmpwi r0, 0
|
|
beq 20b
|
|
HMT_MEDIUM
|
|
21:
|
|
/* Set LPCR. */
|
|
ld r8,VCORE_LPCR(r5)
|
|
mtspr SPRN_LPCR,r8
|
|
isync
|
|
|
|
/*
|
|
* Set the decrementer to the guest decrementer.
|
|
*/
|
|
ld r8,VCPU_DEC_EXPIRES(r4)
|
|
/* r8 is a host timebase value here, convert to guest TB */
|
|
ld r5,HSTATE_KVM_VCORE(r13)
|
|
ld r6,VCORE_TB_OFFSET_APPL(r5)
|
|
add r8,r8,r6
|
|
mftb r7
|
|
subf r3,r7,r8
|
|
mtspr SPRN_DEC,r3
|
|
|
|
/* Check if HDEC expires soon */
|
|
mfspr r3, SPRN_HDEC
|
|
extsw r3, r3
|
|
cmpdi r3, 512 /* 1 microsecond */
|
|
blt hdec_soon
|
|
|
|
/* Clear out and reload the SLB */
|
|
li r6, 0
|
|
slbmte r6, r6
|
|
PPC_SLBIA(6)
|
|
ptesync
|
|
|
|
/* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */
|
|
lwz r5,VCPU_SLB_MAX(r4)
|
|
cmpwi r5,0
|
|
beq 9f
|
|
mtctr r5
|
|
addi r6,r4,VCPU_SLB
|
|
1: ld r8,VCPU_SLB_E(r6)
|
|
ld r9,VCPU_SLB_V(r6)
|
|
slbmte r9,r8
|
|
addi r6,r6,VCPU_SLB_SIZE
|
|
bdnz 1b
|
|
9:
|
|
|
|
deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */
|
|
/* Check if we can deliver an external or decrementer interrupt now */
|
|
ld r0, VCPU_PENDING_EXC(r4)
|
|
cmpdi r0, 0
|
|
beq 71f
|
|
mr r3, r4
|
|
bl kvmppc_guest_entry_inject_int
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
71:
|
|
ld r6, VCPU_SRR0(r4)
|
|
ld r7, VCPU_SRR1(r4)
|
|
mtspr SPRN_SRR0, r6
|
|
mtspr SPRN_SRR1, r7
|
|
|
|
ld r10, VCPU_PC(r4)
|
|
ld r11, VCPU_MSR(r4)
|
|
/* r11 = vcpu->arch.msr & ~MSR_HV */
|
|
rldicl r11, r11, 63 - MSR_HV_LG, 1
|
|
rotldi r11, r11, 1 + MSR_HV_LG
|
|
ori r11, r11, MSR_ME
|
|
|
|
ld r6, VCPU_CTR(r4)
|
|
ld r7, VCPU_XER(r4)
|
|
mtctr r6
|
|
mtxer r7
|
|
|
|
/*
|
|
* Required state:
|
|
* R4 = vcpu
|
|
* R10: value for HSRR0
|
|
* R11: value for HSRR1
|
|
* R13 = PACA
|
|
*/
|
|
fast_guest_return:
|
|
li r0,0
|
|
stb r0,VCPU_CEDED(r4) /* cancel cede */
|
|
mtspr SPRN_HSRR0,r10
|
|
mtspr SPRN_HSRR1,r11
|
|
|
|
/* Activate guest mode, so faults get handled by KVM */
|
|
li r9, KVM_GUEST_MODE_GUEST_HV
|
|
stb r9, HSTATE_IN_GUEST(r13)
|
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
/* Accumulate timing */
|
|
addi r3, r4, VCPU_TB_GUEST
|
|
bl kvmhv_accumulate_time
|
|
#endif
|
|
|
|
/* Enter guest */
|
|
|
|
BEGIN_FTR_SECTION
|
|
ld r5, VCPU_CFAR(r4)
|
|
mtspr SPRN_CFAR, r5
|
|
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
|
|
BEGIN_FTR_SECTION
|
|
ld r0, VCPU_PPR(r4)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
|
|
|
|
ld r5, VCPU_LR(r4)
|
|
mtlr r5
|
|
|
|
ld r1, VCPU_GPR(R1)(r4)
|
|
ld r5, VCPU_GPR(R5)(r4)
|
|
ld r8, VCPU_GPR(R8)(r4)
|
|
ld r9, VCPU_GPR(R9)(r4)
|
|
ld r10, VCPU_GPR(R10)(r4)
|
|
ld r11, VCPU_GPR(R11)(r4)
|
|
ld r12, VCPU_GPR(R12)(r4)
|
|
ld r13, VCPU_GPR(R13)(r4)
|
|
|
|
BEGIN_FTR_SECTION
|
|
mtspr SPRN_PPR, r0
|
|
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
|
|
|
|
ld r6, VCPU_GPR(R6)(r4)
|
|
ld r7, VCPU_GPR(R7)(r4)
|
|
|
|
ld r0, VCPU_CR(r4)
|
|
mtcr r0
|
|
|
|
ld r0, VCPU_GPR(R0)(r4)
|
|
ld r2, VCPU_GPR(R2)(r4)
|
|
ld r3, VCPU_GPR(R3)(r4)
|
|
ld r4, VCPU_GPR(R4)(r4)
|
|
HRFI_TO_GUEST
|
|
b .
|
|
|
|
secondary_too_late:
|
|
li r12, 0
|
|
stw r12, STACK_SLOT_TRAP(r1)
|
|
cmpdi r4, 0
|
|
beq 11f
|
|
stw r12, VCPU_TRAP(r4)
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
addi r3, r4, VCPU_TB_RMEXIT
|
|
bl kvmhv_accumulate_time
|
|
#endif
|
|
11: b kvmhv_switch_to_host
|
|
|
|
no_switch_exit:
|
|
HMT_MEDIUM
|
|
li r12, 0
|
|
b 12f
|
|
hdec_soon:
|
|
li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
|
|
12: stw r12, VCPU_TRAP(r4)
|
|
mr r9, r4
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
addi r3, r4, VCPU_TB_RMEXIT
|
|
bl kvmhv_accumulate_time
|
|
#endif
|
|
b guest_bypass
|
|
|
|
/******************************************************************************
|
|
* *
|
|
* Exit code *
|
|
* *
|
|
*****************************************************************************/
|
|
|
|
/*
|
|
* We come here from the first-level interrupt handlers.
|
|
*/
|
|
.globl kvmppc_interrupt_hv
|
|
kvmppc_interrupt_hv:
|
|
/*
|
|
* Register contents:
|
|
* R9 = HSTATE_IN_GUEST
|
|
* R12 = (guest CR << 32) | interrupt vector
|
|
* R13 = PACA
|
|
* guest R12 saved in shadow VCPU SCRATCH0
|
|
* guest R13 saved in SPRN_SCRATCH0
|
|
* guest R9 saved in HSTATE_SCRATCH2
|
|
*/
|
|
/* We're now back in the host but in guest MMU context */
|
|
cmpwi r9,KVM_GUEST_MODE_HOST_HV
|
|
beq kvmppc_bad_host_intr
|
|
li r9, KVM_GUEST_MODE_HOST_HV
|
|
stb r9, HSTATE_IN_GUEST(r13)
|
|
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
|
|
/* Save registers */
|
|
|
|
std r0, VCPU_GPR(R0)(r9)
|
|
std r1, VCPU_GPR(R1)(r9)
|
|
std r2, VCPU_GPR(R2)(r9)
|
|
std r3, VCPU_GPR(R3)(r9)
|
|
std r4, VCPU_GPR(R4)(r9)
|
|
std r5, VCPU_GPR(R5)(r9)
|
|
std r6, VCPU_GPR(R6)(r9)
|
|
std r7, VCPU_GPR(R7)(r9)
|
|
std r8, VCPU_GPR(R8)(r9)
|
|
ld r0, HSTATE_SCRATCH2(r13)
|
|
std r0, VCPU_GPR(R9)(r9)
|
|
std r10, VCPU_GPR(R10)(r9)
|
|
std r11, VCPU_GPR(R11)(r9)
|
|
ld r3, HSTATE_SCRATCH0(r13)
|
|
std r3, VCPU_GPR(R12)(r9)
|
|
/* CR is in the high half of r12 */
|
|
srdi r4, r12, 32
|
|
std r4, VCPU_CR(r9)
|
|
BEGIN_FTR_SECTION
|
|
ld r3, HSTATE_CFAR(r13)
|
|
std r3, VCPU_CFAR(r9)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
|
|
BEGIN_FTR_SECTION
|
|
ld r4, HSTATE_PPR(r13)
|
|
std r4, VCPU_PPR(r9)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
|
|
|
|
/* Restore R1/R2 so we can handle faults */
|
|
ld r1, HSTATE_HOST_R1(r13)
|
|
ld r2, PACATOC(r13)
|
|
|
|
mfspr r10, SPRN_SRR0
|
|
mfspr r11, SPRN_SRR1
|
|
std r10, VCPU_SRR0(r9)
|
|
std r11, VCPU_SRR1(r9)
|
|
/* trap is in the low half of r12, clear CR from the high half */
|
|
clrldi r12, r12, 32
|
|
andi. r0, r12, 2 /* need to read HSRR0/1? */
|
|
beq 1f
|
|
mfspr r10, SPRN_HSRR0
|
|
mfspr r11, SPRN_HSRR1
|
|
clrrdi r12, r12, 2
|
|
1: std r10, VCPU_PC(r9)
|
|
std r11, VCPU_MSR(r9)
|
|
|
|
GET_SCRATCH0(r3)
|
|
mflr r4
|
|
std r3, VCPU_GPR(R13)(r9)
|
|
std r4, VCPU_LR(r9)
|
|
|
|
stw r12,VCPU_TRAP(r9)
|
|
|
|
/*
|
|
* Now that we have saved away SRR0/1 and HSRR0/1,
|
|
* interrupts are recoverable in principle, so set MSR_RI.
|
|
* This becomes important for relocation-on interrupts from
|
|
* the guest, which we can get in radix mode on POWER9.
|
|
*/
|
|
li r0, MSR_RI
|
|
mtmsrd r0, 1
|
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
addi r3, r9, VCPU_TB_RMINTR
|
|
mr r4, r9
|
|
bl kvmhv_accumulate_time
|
|
ld r5, VCPU_GPR(R5)(r9)
|
|
ld r6, VCPU_GPR(R6)(r9)
|
|
ld r7, VCPU_GPR(R7)(r9)
|
|
ld r8, VCPU_GPR(R8)(r9)
|
|
#endif
|
|
|
|
/* Save HEIR (HV emulation assist reg) in emul_inst
|
|
if this is an HEI (HV emulation interrupt, e40) */
|
|
li r3,KVM_INST_FETCH_FAILED
|
|
stw r3,VCPU_LAST_INST(r9)
|
|
cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
|
|
bne 11f
|
|
mfspr r3,SPRN_HEIR
|
|
11: stw r3,VCPU_HEIR(r9)
|
|
|
|
/* these are volatile across C function calls */
|
|
mfctr r3
|
|
mfxer r4
|
|
std r3, VCPU_CTR(r9)
|
|
std r4, VCPU_XER(r9)
|
|
|
|
/* Save more register state */
|
|
mfdar r3
|
|
mfdsisr r4
|
|
std r3, VCPU_DAR(r9)
|
|
stw r4, VCPU_DSISR(r9)
|
|
|
|
/* If this is a page table miss then see if it's theirs or ours */
|
|
cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
|
|
beq kvmppc_hdsi
|
|
std r3, VCPU_FAULT_DAR(r9)
|
|
stw r4, VCPU_FAULT_DSISR(r9)
|
|
cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
|
|
beq kvmppc_hisi
|
|
|
|
/* See if this is a leftover HDEC interrupt */
|
|
cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
|
|
bne 2f
|
|
mfspr r3,SPRN_HDEC
|
|
extsw r3, r3
|
|
cmpdi r3,0
|
|
mr r4,r9
|
|
bge fast_guest_return
|
|
2:
|
|
/* See if this is an hcall we can handle in real mode */
|
|
cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
|
|
beq hcall_try_real_mode
|
|
|
|
/* Hypervisor doorbell - exit only if host IPI flag set */
|
|
cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
|
|
bne 3f
|
|
lbz r0, HSTATE_HOST_IPI(r13)
|
|
cmpwi r0, 0
|
|
beq maybe_reenter_guest
|
|
b guest_exit_cont
|
|
3:
|
|
/* If it's a hypervisor facility unavailable interrupt, save HFSCR */
|
|
cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL
|
|
bne 14f
|
|
mfspr r3, SPRN_HFSCR
|
|
std r3, VCPU_HFSCR(r9)
|
|
b guest_exit_cont
|
|
14:
|
|
/* External interrupt ? */
|
|
cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
|
|
beq kvmppc_guest_external
|
|
/* See if it is a machine check */
|
|
cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
|
|
beq machine_check_realmode
|
|
/* Or a hypervisor maintenance interrupt */
|
|
cmpwi r12, BOOK3S_INTERRUPT_HMI
|
|
beq hmi_realmode
|
|
|
|
guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
|
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
addi r3, r9, VCPU_TB_RMEXIT
|
|
mr r4, r9
|
|
bl kvmhv_accumulate_time
|
|
#endif
|
|
|
|
/*
|
|
* Possibly flush the link stack here, before we do a blr in
|
|
* kvmhv_switch_to_host.
|
|
*/
|
|
1: nop
|
|
patch_site 1b patch__call_kvm_flush_link_stack
|
|
|
|
/* For hash guest, read the guest SLB and save it away */
|
|
li r5, 0
|
|
lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
|
|
mtctr r0
|
|
li r6,0
|
|
addi r7,r9,VCPU_SLB
|
|
1: slbmfee r8,r6
|
|
andis. r0,r8,SLB_ESID_V@h
|
|
beq 2f
|
|
add r8,r8,r6 /* put index in */
|
|
slbmfev r3,r6
|
|
std r8,VCPU_SLB_E(r7)
|
|
std r3,VCPU_SLB_V(r7)
|
|
addi r7,r7,VCPU_SLB_SIZE
|
|
addi r5,r5,1
|
|
2: addi r6,r6,1
|
|
bdnz 1b
|
|
/* Finally clear out the SLB */
|
|
li r0,0
|
|
slbmte r0,r0
|
|
PPC_SLBIA(6)
|
|
ptesync
|
|
stw r5,VCPU_SLB_MAX(r9)
|
|
|
|
/* load host SLB entries */
|
|
ld r8,PACA_SLBSHADOWPTR(r13)
|
|
|
|
.rept SLB_NUM_BOLTED
|
|
li r3, SLBSHADOW_SAVEAREA
|
|
LDX_BE r5, r8, r3
|
|
addi r3, r3, 8
|
|
LDX_BE r6, r8, r3
|
|
andis. r7,r5,SLB_ESID_V@h
|
|
beq 1f
|
|
slbmte r6,r5
|
|
1: addi r8,r8,16
|
|
.endr
|
|
|
|
guest_bypass:
|
|
stw r12, STACK_SLOT_TRAP(r1)
|
|
|
|
/* Save DEC */
|
|
/* Do this before kvmhv_commence_exit so we know TB is guest TB */
|
|
ld r3, HSTATE_KVM_VCORE(r13)
|
|
mfspr r5,SPRN_DEC
|
|
mftb r6
|
|
extsw r5,r5
|
|
16: add r5,r5,r6
|
|
/* r5 is a guest timebase value here, convert to host TB */
|
|
ld r4,VCORE_TB_OFFSET_APPL(r3)
|
|
subf r5,r4,r5
|
|
std r5,VCPU_DEC_EXPIRES(r9)
|
|
|
|
/* Increment exit count, poke other threads to exit */
|
|
mr r3, r12
|
|
bl kvmhv_commence_exit
|
|
nop
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
|
|
/* Stop others sending VCPU interrupts to this physical CPU */
|
|
li r0, -1
|
|
stw r0, VCPU_CPU(r9)
|
|
stw r0, VCPU_THREAD_CPU(r9)
|
|
|
|
/* Save guest CTRL register, set runlatch to 1 */
|
|
mfspr r6,SPRN_CTRLF
|
|
stw r6,VCPU_CTRL(r9)
|
|
andi. r0,r6,1
|
|
bne 4f
|
|
ori r6,r6,1
|
|
mtspr SPRN_CTRLT,r6
|
|
4:
|
|
/*
|
|
* Save the guest PURR/SPURR
|
|
*/
|
|
mfspr r5,SPRN_PURR
|
|
mfspr r6,SPRN_SPURR
|
|
ld r7,VCPU_PURR(r9)
|
|
ld r8,VCPU_SPURR(r9)
|
|
std r5,VCPU_PURR(r9)
|
|
std r6,VCPU_SPURR(r9)
|
|
subf r5,r7,r5
|
|
subf r6,r8,r6
|
|
|
|
/*
|
|
* Restore host PURR/SPURR and add guest times
|
|
* so that the time in the guest gets accounted.
|
|
*/
|
|
ld r3,HSTATE_PURR(r13)
|
|
ld r4,HSTATE_SPURR(r13)
|
|
add r3,r3,r5
|
|
add r4,r4,r6
|
|
mtspr SPRN_PURR,r3
|
|
mtspr SPRN_SPURR,r4
|
|
|
|
BEGIN_FTR_SECTION
|
|
b 8f
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
|
|
/* Save POWER8-specific registers */
|
|
mfspr r5, SPRN_IAMR
|
|
mfspr r6, SPRN_PSPB
|
|
mfspr r7, SPRN_FSCR
|
|
std r5, VCPU_IAMR(r9)
|
|
stw r6, VCPU_PSPB(r9)
|
|
std r7, VCPU_FSCR(r9)
|
|
mfspr r5, SPRN_IC
|
|
mfspr r7, SPRN_TAR
|
|
std r5, VCPU_IC(r9)
|
|
std r7, VCPU_TAR(r9)
|
|
mfspr r8, SPRN_EBBHR
|
|
std r8, VCPU_EBBHR(r9)
|
|
mfspr r5, SPRN_EBBRR
|
|
mfspr r6, SPRN_BESCR
|
|
mfspr r7, SPRN_PID
|
|
mfspr r8, SPRN_WORT
|
|
std r5, VCPU_EBBRR(r9)
|
|
std r6, VCPU_BESCR(r9)
|
|
stw r7, VCPU_GUEST_PID(r9)
|
|
std r8, VCPU_WORT(r9)
|
|
mfspr r5, SPRN_TCSCR
|
|
mfspr r6, SPRN_ACOP
|
|
mfspr r7, SPRN_CSIGR
|
|
mfspr r8, SPRN_TACR
|
|
std r5, VCPU_TCSCR(r9)
|
|
std r6, VCPU_ACOP(r9)
|
|
std r7, VCPU_CSIGR(r9)
|
|
std r8, VCPU_TACR(r9)
|
|
BEGIN_FTR_SECTION
|
|
ld r5, STACK_SLOT_FSCR(r1)
|
|
mtspr SPRN_FSCR, r5
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
/*
|
|
* Restore various registers to 0, where non-zero values
|
|
* set by the guest could disrupt the host.
|
|
*/
|
|
li r0, 0
|
|
mtspr SPRN_PSPB, r0
|
|
mtspr SPRN_WORT, r0
|
|
mtspr SPRN_TCSCR, r0
|
|
/* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
|
|
li r0, 1
|
|
sldi r0, r0, 31
|
|
mtspr SPRN_MMCRS, r0
|
|
|
|
/* Save and restore AMR, IAMR and UAMOR before turning on the MMU */
|
|
ld r8, STACK_SLOT_IAMR(r1)
|
|
mtspr SPRN_IAMR, r8
|
|
|
|
8: /* Power7 jumps back in here */
|
|
mfspr r5,SPRN_AMR
|
|
mfspr r6,SPRN_UAMOR
|
|
std r5,VCPU_AMR(r9)
|
|
std r6,VCPU_UAMOR(r9)
|
|
ld r5,STACK_SLOT_AMR(r1)
|
|
ld r6,STACK_SLOT_UAMOR(r1)
|
|
mtspr SPRN_AMR, r5
|
|
mtspr SPRN_UAMOR, r6
|
|
|
|
/* Switch DSCR back to host value */
|
|
mfspr r8, SPRN_DSCR
|
|
ld r7, HSTATE_DSCR(r13)
|
|
std r8, VCPU_DSCR(r9)
|
|
mtspr SPRN_DSCR, r7
|
|
|
|
/* Save non-volatile GPRs */
|
|
std r14, VCPU_GPR(R14)(r9)
|
|
std r15, VCPU_GPR(R15)(r9)
|
|
std r16, VCPU_GPR(R16)(r9)
|
|
std r17, VCPU_GPR(R17)(r9)
|
|
std r18, VCPU_GPR(R18)(r9)
|
|
std r19, VCPU_GPR(R19)(r9)
|
|
std r20, VCPU_GPR(R20)(r9)
|
|
std r21, VCPU_GPR(R21)(r9)
|
|
std r22, VCPU_GPR(R22)(r9)
|
|
std r23, VCPU_GPR(R23)(r9)
|
|
std r24, VCPU_GPR(R24)(r9)
|
|
std r25, VCPU_GPR(R25)(r9)
|
|
std r26, VCPU_GPR(R26)(r9)
|
|
std r27, VCPU_GPR(R27)(r9)
|
|
std r28, VCPU_GPR(R28)(r9)
|
|
std r29, VCPU_GPR(R29)(r9)
|
|
std r30, VCPU_GPR(R30)(r9)
|
|
std r31, VCPU_GPR(R31)(r9)
|
|
|
|
/* Save SPRGs */
|
|
mfspr r3, SPRN_SPRG0
|
|
mfspr r4, SPRN_SPRG1
|
|
mfspr r5, SPRN_SPRG2
|
|
mfspr r6, SPRN_SPRG3
|
|
std r3, VCPU_SPRG0(r9)
|
|
std r4, VCPU_SPRG1(r9)
|
|
std r5, VCPU_SPRG2(r9)
|
|
std r6, VCPU_SPRG3(r9)
|
|
|
|
/* save FP state */
|
|
mr r3, r9
|
|
bl kvmppc_save_fp
|
|
|
|
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
|
|
BEGIN_FTR_SECTION
|
|
b 91f
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_TM)
|
|
/*
|
|
* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
|
|
*/
|
|
mr r3, r9
|
|
ld r4, VCPU_MSR(r3)
|
|
li r5, 0 /* don't preserve non-vol regs */
|
|
bl kvmppc_save_tm_hv
|
|
nop
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
91:
|
|
#endif
|
|
|
|
/* Increment yield count if they have a VPA */
|
|
ld r8, VCPU_VPA(r9) /* do they have a VPA? */
|
|
cmpdi r8, 0
|
|
beq 25f
|
|
li r4, LPPACA_YIELDCOUNT
|
|
LWZX_BE r3, r8, r4
|
|
addi r3, r3, 1
|
|
STWX_BE r3, r8, r4
|
|
li r3, 1
|
|
stb r3, VCPU_VPA_DIRTY(r9)
|
|
25:
|
|
/* Save PMU registers if requested */
|
|
/* r8 and cr0.eq are live here */
|
|
mr r3, r9
|
|
li r4, 1
|
|
beq 21f /* if no VPA, save PMU stuff anyway */
|
|
lbz r4, LPPACA_PMCINUSE(r8)
|
|
21: bl kvmhv_save_guest_pmu
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
|
|
/* Restore host values of some registers */
|
|
BEGIN_FTR_SECTION
|
|
ld r5, STACK_SLOT_CIABR(r1)
|
|
ld r6, STACK_SLOT_DAWR0(r1)
|
|
ld r7, STACK_SLOT_DAWRX0(r1)
|
|
mtspr SPRN_CIABR, r5
|
|
/*
|
|
* If the DAWR doesn't work, it's ok to write these here as
|
|
* this value should always be zero
|
|
*/
|
|
mtspr SPRN_DAWR0, r6
|
|
mtspr SPRN_DAWRX0, r7
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
|
|
/*
|
|
* POWER7/POWER8 guest -> host partition switch code.
|
|
* We don't have to lock against tlbies but we do
|
|
* have to coordinate the hardware threads.
|
|
* Here STACK_SLOT_TRAP(r1) contains the trap number.
|
|
*/
|
|
kvmhv_switch_to_host:
|
|
/* Secondary threads wait for primary to do partition switch */
|
|
ld r5,HSTATE_KVM_VCORE(r13)
|
|
ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
|
|
lbz r3,HSTATE_PTID(r13)
|
|
cmpwi r3,0
|
|
beq 15f
|
|
HMT_LOW
|
|
13: lbz r3,VCORE_IN_GUEST(r5)
|
|
cmpwi r3,0
|
|
bne 13b
|
|
HMT_MEDIUM
|
|
b 16f
|
|
|
|
/* Primary thread waits for all the secondaries to exit guest */
|
|
15: lwz r3,VCORE_ENTRY_EXIT(r5)
|
|
rlwinm r0,r3,32-8,0xff
|
|
clrldi r3,r3,56
|
|
cmpw r3,r0
|
|
bne 15b
|
|
isync
|
|
|
|
/* Did we actually switch to the guest at all? */
|
|
lbz r6, VCORE_IN_GUEST(r5)
|
|
cmpwi r6, 0
|
|
beq 19f
|
|
|
|
/* Primary thread switches back to host partition */
|
|
lwz r7,KVM_HOST_LPID(r4)
|
|
ld r6,KVM_HOST_SDR1(r4)
|
|
li r8,LPID_RSVD /* switch to reserved LPID */
|
|
mtspr SPRN_LPID,r8
|
|
ptesync
|
|
mtspr SPRN_SDR1,r6 /* switch to host page table */
|
|
mtspr SPRN_LPID,r7
|
|
isync
|
|
|
|
BEGIN_FTR_SECTION
|
|
/* DPDES and VTB are shared between threads */
|
|
mfspr r7, SPRN_DPDES
|
|
mfspr r8, SPRN_VTB
|
|
std r7, VCORE_DPDES(r5)
|
|
std r8, VCORE_VTB(r5)
|
|
/* clear DPDES so we don't get guest doorbells in the host */
|
|
li r8, 0
|
|
mtspr SPRN_DPDES, r8
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
|
|
/* Subtract timebase offset from timebase */
|
|
ld r8, VCORE_TB_OFFSET_APPL(r5)
|
|
cmpdi r8,0
|
|
beq 17f
|
|
li r0, 0
|
|
std r0, VCORE_TB_OFFSET_APPL(r5)
|
|
mftb r6 /* current guest timebase */
|
|
subf r8,r8,r6
|
|
mtspr SPRN_TBU40,r8 /* update upper 40 bits */
|
|
mftb r7 /* check if lower 24 bits overflowed */
|
|
clrldi r6,r6,40
|
|
clrldi r7,r7,40
|
|
cmpld r7,r6
|
|
bge 17f
|
|
addis r8,r8,0x100 /* if so, increment upper 40 bits */
|
|
mtspr SPRN_TBU40,r8
|
|
|
|
17:
|
|
/*
|
|
* If this is an HMI, we called kvmppc_realmode_hmi_handler
|
|
* above, which may or may not have already called
|
|
* kvmppc_subcore_exit_guest. Fortunately, all that
|
|
* kvmppc_subcore_exit_guest does is clear a flag, so calling
|
|
* it again here is benign even if kvmppc_realmode_hmi_handler
|
|
* has already called it.
|
|
*/
|
|
bl kvmppc_subcore_exit_guest
|
|
nop
|
|
30: ld r5,HSTATE_KVM_VCORE(r13)
|
|
ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
|
|
|
|
/* Reset PCR */
|
|
ld r0, VCORE_PCR(r5)
|
|
LOAD_REG_IMMEDIATE(r6, PCR_MASK)
|
|
cmpld r0, r6
|
|
beq 18f
|
|
mtspr SPRN_PCR, r6
|
|
18:
|
|
/* Signal secondary CPUs to continue */
|
|
li r0, 0
|
|
stb r0,VCORE_IN_GUEST(r5)
|
|
19: lis r8,0x7fff /* MAX_INT@h */
|
|
mtspr SPRN_HDEC,r8
|
|
|
|
16: ld r8,KVM_HOST_LPCR(r4)
|
|
mtspr SPRN_LPCR,r8
|
|
isync
|
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
/* Finish timing, if we have a vcpu */
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
cmpdi r4, 0
|
|
li r3, 0
|
|
beq 2f
|
|
bl kvmhv_accumulate_time
|
|
2:
|
|
#endif
|
|
/* Unset guest mode */
|
|
li r0, KVM_GUEST_MODE_NONE
|
|
stb r0, HSTATE_IN_GUEST(r13)
|
|
|
|
lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */
|
|
ld r0, SFS+PPC_LR_STKOFF(r1)
|
|
addi r1, r1, SFS
|
|
mtlr r0
|
|
blr
|
|
|
|
.balign 32
|
|
.global kvm_flush_link_stack
|
|
kvm_flush_link_stack:
|
|
/* Save LR into r0 */
|
|
mflr r0
|
|
|
|
/* Flush the link stack. On Power8 it's up to 32 entries in size. */
|
|
.rept 32
|
|
bl .+4
|
|
.endr
|
|
|
|
/* And on Power9 it's up to 64. */
|
|
BEGIN_FTR_SECTION
|
|
.rept 32
|
|
bl .+4
|
|
.endr
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
|
|
|
|
/* Restore LR */
|
|
mtlr r0
|
|
blr
|
|
|
|
kvmppc_guest_external:
|
|
/* External interrupt, first check for host_ipi. If this is
|
|
* set, we know the host wants us out so let's do it now
|
|
*/
|
|
bl kvmppc_read_intr
|
|
|
|
/*
|
|
* Restore the active volatile registers after returning from
|
|
* a C function.
|
|
*/
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
li r12, BOOK3S_INTERRUPT_EXTERNAL
|
|
|
|
/*
|
|
* kvmppc_read_intr return codes:
|
|
*
|
|
* Exit to host (r3 > 0)
|
|
* 1 An interrupt is pending that needs to be handled by the host
|
|
* Exit guest and return to host by branching to guest_exit_cont
|
|
*
|
|
* 2 Passthrough that needs completion in the host
|
|
* Exit guest and return to host by branching to guest_exit_cont
|
|
* However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD
|
|
* to indicate to the host to complete handling the interrupt
|
|
*
|
|
* Before returning to guest, we check if any CPU is heading out
|
|
* to the host and if so, we head out also. If no CPUs are heading
|
|
* check return values <= 0.
|
|
*
|
|
* Return to guest (r3 <= 0)
|
|
* 0 No external interrupt is pending
|
|
* -1 A guest wakeup IPI (which has now been cleared)
|
|
* In either case, we return to guest to deliver any pending
|
|
* guest interrupts.
|
|
*
|
|
* -2 A PCI passthrough external interrupt was handled
|
|
* (interrupt was delivered directly to guest)
|
|
* Return to guest to deliver any pending guest interrupts.
|
|
*/
|
|
|
|
cmpdi r3, 1
|
|
ble 1f
|
|
|
|
/* Return code = 2 */
|
|
li r12, BOOK3S_INTERRUPT_HV_RM_HARD
|
|
stw r12, VCPU_TRAP(r9)
|
|
b guest_exit_cont
|
|
|
|
1: /* Return code <= 1 */
|
|
cmpdi r3, 0
|
|
bgt guest_exit_cont
|
|
|
|
/* Return code <= 0 */
|
|
maybe_reenter_guest:
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
lwz r0, VCORE_ENTRY_EXIT(r5)
|
|
cmpwi r0, 0x100
|
|
mr r4, r9
|
|
blt deliver_guest_interrupt
|
|
b guest_exit_cont
|
|
|
|
/*
|
|
* Check whether an HDSI is an HPTE not found fault or something else.
|
|
* If it is an HPTE not found fault that is due to the guest accessing
|
|
* a page that they have mapped but which we have paged out, then
|
|
* we continue on with the guest exit path. In all other cases,
|
|
* reflect the HDSI to the guest as a DSI.
|
|
*/
|
|
kvmppc_hdsi:
|
|
mfspr r4, SPRN_HDAR
|
|
mfspr r6, SPRN_HDSISR
|
|
/* HPTE not found fault or protection fault? */
|
|
andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
|
|
beq 1f /* if not, send it to the guest */
|
|
andi. r0, r11, MSR_DR /* data relocation enabled? */
|
|
beq 3f
|
|
clrrdi r0, r4, 28
|
|
PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
|
|
li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
|
|
bne 7f /* if no SLB entry found */
|
|
4: std r4, VCPU_FAULT_DAR(r9)
|
|
stw r6, VCPU_FAULT_DSISR(r9)
|
|
|
|
/* Search the hash table. */
|
|
mr r3, r9 /* vcpu pointer */
|
|
li r7, 1 /* data fault */
|
|
bl kvmppc_hpte_hv_fault
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
ld r10, VCPU_PC(r9)
|
|
ld r11, VCPU_MSR(r9)
|
|
li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
|
|
cmpdi r3, 0 /* retry the instruction */
|
|
beq 6f
|
|
cmpdi r3, -1 /* handle in kernel mode */
|
|
beq guest_exit_cont
|
|
cmpdi r3, -2 /* MMIO emulation; need instr word */
|
|
beq 2f
|
|
|
|
/* Synthesize a DSI (or DSegI) for the guest */
|
|
ld r4, VCPU_FAULT_DAR(r9)
|
|
mr r6, r3
|
|
1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE
|
|
mtspr SPRN_DSISR, r6
|
|
7: mtspr SPRN_DAR, r4
|
|
mtspr SPRN_SRR0, r10
|
|
mtspr SPRN_SRR1, r11
|
|
mr r10, r0
|
|
bl kvmppc_msr_interrupt
|
|
fast_interrupt_c_return:
|
|
6: ld r7, VCPU_CTR(r9)
|
|
ld r8, VCPU_XER(r9)
|
|
mtctr r7
|
|
mtxer r8
|
|
mr r4, r9
|
|
b fast_guest_return
|
|
|
|
3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
|
|
ld r5, KVM_VRMA_SLB_V(r5)
|
|
b 4b
|
|
|
|
/* If this is for emulated MMIO, load the instruction word */
|
|
2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
|
|
|
|
/* Set guest mode to 'jump over instruction' so if lwz faults
|
|
* we'll just continue at the next IP. */
|
|
li r0, KVM_GUEST_MODE_SKIP
|
|
stb r0, HSTATE_IN_GUEST(r13)
|
|
|
|
/* Do the access with MSR:DR enabled */
|
|
mfmsr r3
|
|
ori r4, r3, MSR_DR /* Enable paging for data */
|
|
mtmsrd r4
|
|
lwz r8, 0(r10)
|
|
mtmsrd r3
|
|
|
|
/* Store the result */
|
|
stw r8, VCPU_LAST_INST(r9)
|
|
|
|
/* Unset guest mode. */
|
|
li r0, KVM_GUEST_MODE_HOST_HV
|
|
stb r0, HSTATE_IN_GUEST(r13)
|
|
b guest_exit_cont
|
|
|
|
/*
|
|
* Similarly for an HISI, reflect it to the guest as an ISI unless
|
|
* it is an HPTE not found fault for a page that we have paged out.
|
|
*/
|
|
kvmppc_hisi:
|
|
andis. r0, r11, SRR1_ISI_NOPT@h
|
|
beq 1f
|
|
andi. r0, r11, MSR_IR /* instruction relocation enabled? */
|
|
beq 3f
|
|
clrrdi r0, r10, 28
|
|
PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
|
|
li r0, BOOK3S_INTERRUPT_INST_SEGMENT
|
|
bne 7f /* if no SLB entry found */
|
|
4:
|
|
/* Search the hash table. */
|
|
mr r3, r9 /* vcpu pointer */
|
|
mr r4, r10
|
|
mr r6, r11
|
|
li r7, 0 /* instruction fault */
|
|
bl kvmppc_hpte_hv_fault
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
ld r10, VCPU_PC(r9)
|
|
ld r11, VCPU_MSR(r9)
|
|
li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
|
|
cmpdi r3, 0 /* retry the instruction */
|
|
beq fast_interrupt_c_return
|
|
cmpdi r3, -1 /* handle in kernel mode */
|
|
beq guest_exit_cont
|
|
|
|
/* Synthesize an ISI (or ISegI) for the guest */
|
|
mr r11, r3
|
|
1: li r0, BOOK3S_INTERRUPT_INST_STORAGE
|
|
7: mtspr SPRN_SRR0, r10
|
|
mtspr SPRN_SRR1, r11
|
|
mr r10, r0
|
|
bl kvmppc_msr_interrupt
|
|
b fast_interrupt_c_return
|
|
|
|
3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
|
|
ld r5, KVM_VRMA_SLB_V(r6)
|
|
b 4b
|
|
|
|
/*
|
|
* Try to handle an hcall in real mode.
|
|
* Returns to the guest if we handle it, or continues on up to
|
|
* the kernel if we can't (i.e. if we don't have a handler for
|
|
* it, or if the handler returns H_TOO_HARD).
|
|
*
|
|
* r5 - r8 contain hcall args,
|
|
* r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
|
|
*/
|
|
hcall_try_real_mode:
|
|
ld r3,VCPU_GPR(R3)(r9)
|
|
andi. r0,r11,MSR_PR
|
|
/* sc 1 from userspace - reflect to guest syscall */
|
|
bne sc_1_fast_return
|
|
clrrdi r3,r3,2
|
|
cmpldi r3,hcall_real_table_end - hcall_real_table
|
|
bge guest_exit_cont
|
|
/* See if this hcall is enabled for in-kernel handling */
|
|
ld r4, VCPU_KVM(r9)
|
|
srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */
|
|
sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */
|
|
add r4, r4, r0
|
|
ld r0, KVM_ENABLED_HCALLS(r4)
|
|
rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */
|
|
srd r0, r0, r4
|
|
andi. r0, r0, 1
|
|
beq guest_exit_cont
|
|
/* Get pointer to handler, if any, and call it */
|
|
LOAD_REG_ADDR(r4, hcall_real_table)
|
|
lwax r3,r3,r4
|
|
cmpwi r3,0
|
|
beq guest_exit_cont
|
|
add r12,r3,r4
|
|
mtctr r12
|
|
mr r3,r9 /* get vcpu pointer */
|
|
ld r4,VCPU_GPR(R4)(r9)
|
|
bctrl
|
|
cmpdi r3,H_TOO_HARD
|
|
beq hcall_real_fallback
|
|
ld r4,HSTATE_KVM_VCPU(r13)
|
|
std r3,VCPU_GPR(R3)(r4)
|
|
ld r10,VCPU_PC(r4)
|
|
ld r11,VCPU_MSR(r4)
|
|
b fast_guest_return
|
|
|
|
sc_1_fast_return:
|
|
mtspr SPRN_SRR0,r10
|
|
mtspr SPRN_SRR1,r11
|
|
li r10, BOOK3S_INTERRUPT_SYSCALL
|
|
bl kvmppc_msr_interrupt
|
|
mr r4,r9
|
|
b fast_guest_return
|
|
|
|
/* We've attempted a real mode hcall, but it's punted it back
|
|
* to userspace. We need to restore some clobbered volatiles
|
|
* before resuming the pass-it-to-qemu path */
|
|
hcall_real_fallback:
|
|
li r12,BOOK3S_INTERRUPT_SYSCALL
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
|
|
b guest_exit_cont
|
|
|
|
.globl hcall_real_table
|
|
hcall_real_table:
|
|
.long 0 /* 0 - unused */
|
|
.long DOTSYM(kvmppc_h_remove) - hcall_real_table
|
|
.long DOTSYM(kvmppc_h_enter) - hcall_real_table
|
|
.long DOTSYM(kvmppc_h_read) - hcall_real_table
|
|
.long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
|
|
.long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
|
|
.long DOTSYM(kvmppc_h_protect) - hcall_real_table
|
|
#ifdef CONFIG_SPAPR_TCE_IOMMU
|
|
.long DOTSYM(kvmppc_h_get_tce) - hcall_real_table
|
|
.long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table
|
|
#else
|
|
.long 0 /* 0x1c */
|
|
.long 0 /* 0x20 */
|
|
#endif
|
|
.long 0 /* 0x24 - H_SET_SPRG0 */
|
|
.long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
|
|
.long DOTSYM(kvmppc_rm_h_page_init) - hcall_real_table
|
|
.long 0 /* 0x30 */
|
|
.long 0 /* 0x34 */
|
|
.long 0 /* 0x38 */
|
|
.long 0 /* 0x3c */
|
|
.long 0 /* 0x40 */
|
|
.long 0 /* 0x44 */
|
|
.long 0 /* 0x48 */
|
|
.long 0 /* 0x4c */
|
|
.long 0 /* 0x50 */
|
|
.long 0 /* 0x54 */
|
|
.long 0 /* 0x58 */
|
|
.long 0 /* 0x5c */
|
|
.long 0 /* 0x60 */
|
|
#ifdef CONFIG_KVM_XICS
|
|
.long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
|
|
.long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
|
|
.long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
|
|
.long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table
|
|
.long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
|
|
#else
|
|
.long 0 /* 0x64 - H_EOI */
|
|
.long 0 /* 0x68 - H_CPPR */
|
|
.long 0 /* 0x6c - H_IPI */
|
|
.long 0 /* 0x70 - H_IPOLL */
|
|
.long 0 /* 0x74 - H_XIRR */
|
|
#endif
|
|
.long 0 /* 0x78 */
|
|
.long 0 /* 0x7c */
|
|
.long 0 /* 0x80 */
|
|
.long 0 /* 0x84 */
|
|
.long 0 /* 0x88 */
|
|
.long 0 /* 0x8c */
|
|
.long 0 /* 0x90 */
|
|
.long 0 /* 0x94 */
|
|
.long 0 /* 0x98 */
|
|
.long 0 /* 0x9c */
|
|
.long 0 /* 0xa0 */
|
|
.long 0 /* 0xa4 */
|
|
.long 0 /* 0xa8 */
|
|
.long 0 /* 0xac */
|
|
.long 0 /* 0xb0 */
|
|
.long 0 /* 0xb4 */
|
|
.long 0 /* 0xb8 */
|
|
.long 0 /* 0xbc */
|
|
.long 0 /* 0xc0 */
|
|
.long 0 /* 0xc4 */
|
|
.long 0 /* 0xc8 */
|
|
.long 0 /* 0xcc */
|
|
.long 0 /* 0xd0 */
|
|
.long 0 /* 0xd4 */
|
|
.long 0 /* 0xd8 */
|
|
.long 0 /* 0xdc */
|
|
.long DOTSYM(kvmppc_h_cede) - hcall_real_table
|
|
.long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
|
|
.long 0 /* 0xe8 */
|
|
.long 0 /* 0xec */
|
|
.long 0 /* 0xf0 */
|
|
.long 0 /* 0xf4 */
|
|
.long 0 /* 0xf8 */
|
|
.long 0 /* 0xfc */
|
|
.long 0 /* 0x100 */
|
|
.long 0 /* 0x104 */
|
|
.long 0 /* 0x108 */
|
|
.long 0 /* 0x10c */
|
|
.long 0 /* 0x110 */
|
|
.long 0 /* 0x114 */
|
|
.long 0 /* 0x118 */
|
|
.long 0 /* 0x11c */
|
|
.long 0 /* 0x120 */
|
|
.long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
|
|
.long 0 /* 0x128 */
|
|
.long 0 /* 0x12c */
|
|
.long 0 /* 0x130 */
|
|
.long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
|
|
#ifdef CONFIG_SPAPR_TCE_IOMMU
|
|
.long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table
|
|
.long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table
|
|
#else
|
|
.long 0 /* 0x138 */
|
|
.long 0 /* 0x13c */
|
|
#endif
|
|
.long 0 /* 0x140 */
|
|
.long 0 /* 0x144 */
|
|
.long 0 /* 0x148 */
|
|
.long 0 /* 0x14c */
|
|
.long 0 /* 0x150 */
|
|
.long 0 /* 0x154 */
|
|
.long 0 /* 0x158 */
|
|
.long 0 /* 0x15c */
|
|
.long 0 /* 0x160 */
|
|
.long 0 /* 0x164 */
|
|
.long 0 /* 0x168 */
|
|
.long 0 /* 0x16c */
|
|
.long 0 /* 0x170 */
|
|
.long 0 /* 0x174 */
|
|
.long 0 /* 0x178 */
|
|
.long 0 /* 0x17c */
|
|
.long 0 /* 0x180 */
|
|
.long 0 /* 0x184 */
|
|
.long 0 /* 0x188 */
|
|
.long 0 /* 0x18c */
|
|
.long 0 /* 0x190 */
|
|
.long 0 /* 0x194 */
|
|
.long 0 /* 0x198 */
|
|
.long 0 /* 0x19c */
|
|
.long 0 /* 0x1a0 */
|
|
.long 0 /* 0x1a4 */
|
|
.long 0 /* 0x1a8 */
|
|
.long 0 /* 0x1ac */
|
|
.long 0 /* 0x1b0 */
|
|
.long 0 /* 0x1b4 */
|
|
.long 0 /* 0x1b8 */
|
|
.long 0 /* 0x1bc */
|
|
.long 0 /* 0x1c0 */
|
|
.long 0 /* 0x1c4 */
|
|
.long 0 /* 0x1c8 */
|
|
.long 0 /* 0x1cc */
|
|
.long 0 /* 0x1d0 */
|
|
.long 0 /* 0x1d4 */
|
|
.long 0 /* 0x1d8 */
|
|
.long 0 /* 0x1dc */
|
|
.long 0 /* 0x1e0 */
|
|
.long 0 /* 0x1e4 */
|
|
.long 0 /* 0x1e8 */
|
|
.long 0 /* 0x1ec */
|
|
.long 0 /* 0x1f0 */
|
|
.long 0 /* 0x1f4 */
|
|
.long 0 /* 0x1f8 */
|
|
.long 0 /* 0x1fc */
|
|
.long 0 /* 0x200 */
|
|
.long 0 /* 0x204 */
|
|
.long 0 /* 0x208 */
|
|
.long 0 /* 0x20c */
|
|
.long 0 /* 0x210 */
|
|
.long 0 /* 0x214 */
|
|
.long 0 /* 0x218 */
|
|
.long 0 /* 0x21c */
|
|
.long 0 /* 0x220 */
|
|
.long 0 /* 0x224 */
|
|
.long 0 /* 0x228 */
|
|
.long 0 /* 0x22c */
|
|
.long 0 /* 0x230 */
|
|
.long 0 /* 0x234 */
|
|
.long 0 /* 0x238 */
|
|
.long 0 /* 0x23c */
|
|
.long 0 /* 0x240 */
|
|
.long 0 /* 0x244 */
|
|
.long 0 /* 0x248 */
|
|
.long 0 /* 0x24c */
|
|
.long 0 /* 0x250 */
|
|
.long 0 /* 0x254 */
|
|
.long 0 /* 0x258 */
|
|
.long 0 /* 0x25c */
|
|
.long 0 /* 0x260 */
|
|
.long 0 /* 0x264 */
|
|
.long 0 /* 0x268 */
|
|
.long 0 /* 0x26c */
|
|
.long 0 /* 0x270 */
|
|
.long 0 /* 0x274 */
|
|
.long 0 /* 0x278 */
|
|
.long 0 /* 0x27c */
|
|
.long 0 /* 0x280 */
|
|
.long 0 /* 0x284 */
|
|
.long 0 /* 0x288 */
|
|
.long 0 /* 0x28c */
|
|
.long 0 /* 0x290 */
|
|
.long 0 /* 0x294 */
|
|
.long 0 /* 0x298 */
|
|
.long 0 /* 0x29c */
|
|
.long 0 /* 0x2a0 */
|
|
.long 0 /* 0x2a4 */
|
|
.long 0 /* 0x2a8 */
|
|
.long 0 /* 0x2ac */
|
|
.long 0 /* 0x2b0 */
|
|
.long 0 /* 0x2b4 */
|
|
.long 0 /* 0x2b8 */
|
|
.long 0 /* 0x2bc */
|
|
.long 0 /* 0x2c0 */
|
|
.long 0 /* 0x2c4 */
|
|
.long 0 /* 0x2c8 */
|
|
.long 0 /* 0x2cc */
|
|
.long 0 /* 0x2d0 */
|
|
.long 0 /* 0x2d4 */
|
|
.long 0 /* 0x2d8 */
|
|
.long 0 /* 0x2dc */
|
|
.long 0 /* 0x2e0 */
|
|
.long 0 /* 0x2e4 */
|
|
.long 0 /* 0x2e8 */
|
|
.long 0 /* 0x2ec */
|
|
.long 0 /* 0x2f0 */
|
|
.long 0 /* 0x2f4 */
|
|
.long 0 /* 0x2f8 */
|
|
#ifdef CONFIG_KVM_XICS
|
|
.long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table
|
|
#else
|
|
.long 0 /* 0x2fc - H_XIRR_X*/
|
|
#endif
|
|
.long DOTSYM(kvmppc_rm_h_random) - hcall_real_table
|
|
.globl hcall_real_table_end
|
|
hcall_real_table_end:
|
|
|
|
_GLOBAL(kvmppc_h_set_xdabr)
|
|
EXPORT_SYMBOL_GPL(kvmppc_h_set_xdabr)
|
|
andi. r0, r5, DABRX_USER | DABRX_KERNEL
|
|
beq 6f
|
|
li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
|
|
andc. r0, r5, r0
|
|
beq 3f
|
|
6: li r3, H_PARAMETER
|
|
blr
|
|
|
|
_GLOBAL(kvmppc_h_set_dabr)
|
|
EXPORT_SYMBOL_GPL(kvmppc_h_set_dabr)
|
|
li r5, DABRX_USER | DABRX_KERNEL
|
|
3:
|
|
BEGIN_FTR_SECTION
|
|
b 2f
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
std r4,VCPU_DABR(r3)
|
|
stw r5, VCPU_DABRX(r3)
|
|
mtspr SPRN_DABRX, r5
|
|
/* Work around P7 bug where DABR can get corrupted on mtspr */
|
|
1: mtspr SPRN_DABR,r4
|
|
mfspr r5, SPRN_DABR
|
|
cmpd r4, r5
|
|
bne 1b
|
|
isync
|
|
li r3,0
|
|
blr
|
|
|
|
2:
|
|
LOAD_REG_ADDR(r11, dawr_force_enable)
|
|
lbz r11, 0(r11)
|
|
cmpdi r11, 0
|
|
bne 3f
|
|
li r3, H_HARDWARE
|
|
blr
|
|
3:
|
|
/* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
|
|
rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
|
|
rlwimi r5, r4, 2, DAWRX_WT
|
|
clrrdi r4, r4, 3
|
|
std r4, VCPU_DAWR0(r3)
|
|
std r5, VCPU_DAWRX0(r3)
|
|
/*
|
|
* If came in through the real mode hcall handler then it is necessary
|
|
* to write the registers since the return path won't. Otherwise it is
|
|
* sufficient to store then in the vcpu struct as they will be loaded
|
|
* next time the vcpu is run.
|
|
*/
|
|
mfmsr r6
|
|
andi. r6, r6, MSR_DR /* in real mode? */
|
|
bne 4f
|
|
mtspr SPRN_DAWR0, r4
|
|
mtspr SPRN_DAWRX0, r5
|
|
4: li r3, 0
|
|
blr
|
|
|
|
_GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */
|
|
ori r11,r11,MSR_EE
|
|
std r11,VCPU_MSR(r3)
|
|
li r0,1
|
|
stb r0,VCPU_CEDED(r3)
|
|
sync /* order setting ceded vs. testing prodded */
|
|
lbz r5,VCPU_PRODDED(r3)
|
|
cmpwi r5,0
|
|
bne kvm_cede_prodded
|
|
li r12,0 /* set trap to 0 to say hcall is handled */
|
|
stw r12,VCPU_TRAP(r3)
|
|
li r0,H_SUCCESS
|
|
std r0,VCPU_GPR(R3)(r3)
|
|
|
|
/*
|
|
* Set our bit in the bitmask of napping threads unless all the
|
|
* other threads are already napping, in which case we send this
|
|
* up to the host.
|
|
*/
|
|
ld r5,HSTATE_KVM_VCORE(r13)
|
|
lbz r6,HSTATE_PTID(r13)
|
|
lwz r8,VCORE_ENTRY_EXIT(r5)
|
|
clrldi r8,r8,56
|
|
li r0,1
|
|
sld r0,r0,r6
|
|
addi r6,r5,VCORE_NAPPING_THREADS
|
|
31: lwarx r4,0,r6
|
|
or r4,r4,r0
|
|
cmpw r4,r8
|
|
beq kvm_cede_exit
|
|
stwcx. r4,0,r6
|
|
bne 31b
|
|
/* order napping_threads update vs testing entry_exit_map */
|
|
isync
|
|
li r0,NAPPING_CEDE
|
|
stb r0,HSTATE_NAPPING(r13)
|
|
lwz r7,VCORE_ENTRY_EXIT(r5)
|
|
cmpwi r7,0x100
|
|
bge 33f /* another thread already exiting */
|
|
|
|
/*
|
|
* Although not specifically required by the architecture, POWER7
|
|
* preserves the following registers in nap mode, even if an SMT mode
|
|
* switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
|
|
* DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
|
|
*/
|
|
/* Save non-volatile GPRs */
|
|
std r14, VCPU_GPR(R14)(r3)
|
|
std r15, VCPU_GPR(R15)(r3)
|
|
std r16, VCPU_GPR(R16)(r3)
|
|
std r17, VCPU_GPR(R17)(r3)
|
|
std r18, VCPU_GPR(R18)(r3)
|
|
std r19, VCPU_GPR(R19)(r3)
|
|
std r20, VCPU_GPR(R20)(r3)
|
|
std r21, VCPU_GPR(R21)(r3)
|
|
std r22, VCPU_GPR(R22)(r3)
|
|
std r23, VCPU_GPR(R23)(r3)
|
|
std r24, VCPU_GPR(R24)(r3)
|
|
std r25, VCPU_GPR(R25)(r3)
|
|
std r26, VCPU_GPR(R26)(r3)
|
|
std r27, VCPU_GPR(R27)(r3)
|
|
std r28, VCPU_GPR(R28)(r3)
|
|
std r29, VCPU_GPR(R29)(r3)
|
|
std r30, VCPU_GPR(R30)(r3)
|
|
std r31, VCPU_GPR(R31)(r3)
|
|
|
|
/* save FP state */
|
|
bl kvmppc_save_fp
|
|
|
|
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
|
|
BEGIN_FTR_SECTION
|
|
b 91f
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_TM)
|
|
/*
|
|
* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
|
|
*/
|
|
ld r3, HSTATE_KVM_VCPU(r13)
|
|
ld r4, VCPU_MSR(r3)
|
|
li r5, 0 /* don't preserve non-vol regs */
|
|
bl kvmppc_save_tm_hv
|
|
nop
|
|
91:
|
|
#endif
|
|
|
|
/*
|
|
* Set DEC to the smaller of DEC and HDEC, so that we wake
|
|
* no later than the end of our timeslice (HDEC interrupts
|
|
* don't wake us from nap).
|
|
*/
|
|
mfspr r3, SPRN_DEC
|
|
mfspr r4, SPRN_HDEC
|
|
mftb r5
|
|
extsw r3, r3
|
|
extsw r4, r4
|
|
cmpd r3, r4
|
|
ble 67f
|
|
mtspr SPRN_DEC, r4
|
|
67:
|
|
/* save expiry time of guest decrementer */
|
|
add r3, r3, r5
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
ld r6, VCORE_TB_OFFSET_APPL(r5)
|
|
subf r3, r6, r3 /* convert to host TB value */
|
|
std r3, VCPU_DEC_EXPIRES(r4)
|
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
addi r3, r4, VCPU_TB_CEDE
|
|
bl kvmhv_accumulate_time
|
|
#endif
|
|
|
|
lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */
|
|
|
|
/* Go back to host stack */
|
|
ld r1, HSTATE_HOST_R1(r13)
|
|
|
|
/*
|
|
* Take a nap until a decrementer or external or doobell interrupt
|
|
* occurs, with PECE1 and PECE0 set in LPCR.
|
|
* On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
|
|
* Also clear the runlatch bit before napping.
|
|
*/
|
|
kvm_do_nap:
|
|
mfspr r0, SPRN_CTRLF
|
|
clrrdi r0, r0, 1
|
|
mtspr SPRN_CTRLT, r0
|
|
|
|
li r0,1
|
|
stb r0,HSTATE_HWTHREAD_REQ(r13)
|
|
mfspr r5,SPRN_LPCR
|
|
ori r5,r5,LPCR_PECE0 | LPCR_PECE1
|
|
BEGIN_FTR_SECTION
|
|
ori r5, r5, LPCR_PECEDH
|
|
rlwimi r5, r3, 0, LPCR_PECEDP
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
|
|
kvm_nap_sequence: /* desired LPCR value in r5 */
|
|
li r3, PNV_THREAD_NAP
|
|
mtspr SPRN_LPCR,r5
|
|
isync
|
|
|
|
bl isa206_idle_insn_mayloss
|
|
|
|
mfspr r0, SPRN_CTRLF
|
|
ori r0, r0, 1
|
|
mtspr SPRN_CTRLT, r0
|
|
|
|
mtspr SPRN_SRR1, r3
|
|
|
|
li r0, 0
|
|
stb r0, PACA_FTRACE_ENABLED(r13)
|
|
|
|
li r0, KVM_HWTHREAD_IN_KVM
|
|
stb r0, HSTATE_HWTHREAD_STATE(r13)
|
|
|
|
lbz r0, HSTATE_NAPPING(r13)
|
|
cmpwi r0, NAPPING_CEDE
|
|
beq kvm_end_cede
|
|
cmpwi r0, NAPPING_NOVCPU
|
|
beq kvm_novcpu_wakeup
|
|
cmpwi r0, NAPPING_UNSPLIT
|
|
beq kvm_unsplit_wakeup
|
|
twi 31,0,0 /* Nap state must not be zero */
|
|
|
|
33: mr r4, r3
|
|
li r3, 0
|
|
li r12, 0
|
|
b 34f
|
|
|
|
kvm_end_cede:
|
|
/* Woken by external or decrementer interrupt */
|
|
|
|
/* get vcpu pointer */
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
addi r3, r4, VCPU_TB_RMINTR
|
|
bl kvmhv_accumulate_time
|
|
#endif
|
|
|
|
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
|
|
BEGIN_FTR_SECTION
|
|
b 91f
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_TM)
|
|
/*
|
|
* NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR)
|
|
*/
|
|
mr r3, r4
|
|
ld r4, VCPU_MSR(r3)
|
|
li r5, 0 /* don't preserve non-vol regs */
|
|
bl kvmppc_restore_tm_hv
|
|
nop
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
91:
|
|
#endif
|
|
|
|
/* load up FP state */
|
|
bl kvmppc_load_fp
|
|
|
|
/* Restore guest decrementer */
|
|
ld r3, VCPU_DEC_EXPIRES(r4)
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
ld r6, VCORE_TB_OFFSET_APPL(r5)
|
|
add r3, r3, r6 /* convert host TB to guest TB value */
|
|
mftb r7
|
|
subf r3, r7, r3
|
|
mtspr SPRN_DEC, r3
|
|
|
|
/* Load NV GPRS */
|
|
ld r14, VCPU_GPR(R14)(r4)
|
|
ld r15, VCPU_GPR(R15)(r4)
|
|
ld r16, VCPU_GPR(R16)(r4)
|
|
ld r17, VCPU_GPR(R17)(r4)
|
|
ld r18, VCPU_GPR(R18)(r4)
|
|
ld r19, VCPU_GPR(R19)(r4)
|
|
ld r20, VCPU_GPR(R20)(r4)
|
|
ld r21, VCPU_GPR(R21)(r4)
|
|
ld r22, VCPU_GPR(R22)(r4)
|
|
ld r23, VCPU_GPR(R23)(r4)
|
|
ld r24, VCPU_GPR(R24)(r4)
|
|
ld r25, VCPU_GPR(R25)(r4)
|
|
ld r26, VCPU_GPR(R26)(r4)
|
|
ld r27, VCPU_GPR(R27)(r4)
|
|
ld r28, VCPU_GPR(R28)(r4)
|
|
ld r29, VCPU_GPR(R29)(r4)
|
|
ld r30, VCPU_GPR(R30)(r4)
|
|
ld r31, VCPU_GPR(R31)(r4)
|
|
|
|
/* Check the wake reason in SRR1 to see why we got here */
|
|
bl kvmppc_check_wake_reason
|
|
|
|
/*
|
|
* Restore volatile registers since we could have called a
|
|
* C routine in kvmppc_check_wake_reason
|
|
* r4 = VCPU
|
|
* r3 tells us whether we need to return to host or not
|
|
* WARNING: it gets checked further down:
|
|
* should not modify r3 until this check is done.
|
|
*/
|
|
ld r4, HSTATE_KVM_VCPU(r13)
|
|
|
|
/* clear our bit in vcore->napping_threads */
|
|
34: ld r5,HSTATE_KVM_VCORE(r13)
|
|
lbz r7,HSTATE_PTID(r13)
|
|
li r0,1
|
|
sld r0,r0,r7
|
|
addi r6,r5,VCORE_NAPPING_THREADS
|
|
32: lwarx r7,0,r6
|
|
andc r7,r7,r0
|
|
stwcx. r7,0,r6
|
|
bne 32b
|
|
li r0,0
|
|
stb r0,HSTATE_NAPPING(r13)
|
|
|
|
/* See if the wake reason saved in r3 means we need to exit */
|
|
stw r12, VCPU_TRAP(r4)
|
|
mr r9, r4
|
|
cmpdi r3, 0
|
|
bgt guest_exit_cont
|
|
b maybe_reenter_guest
|
|
|
|
/* cede when already previously prodded case */
|
|
kvm_cede_prodded:
|
|
li r0,0
|
|
stb r0,VCPU_PRODDED(r3)
|
|
sync /* order testing prodded vs. clearing ceded */
|
|
stb r0,VCPU_CEDED(r3)
|
|
li r3,H_SUCCESS
|
|
blr
|
|
|
|
/* we've ceded but we want to give control to the host */
|
|
kvm_cede_exit:
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
b guest_exit_cont
|
|
|
|
/* Try to do machine check recovery in real mode */
|
|
machine_check_realmode:
|
|
mr r3, r9 /* get vcpu pointer */
|
|
bl kvmppc_realmode_machine_check
|
|
nop
|
|
/* all machine checks go to virtual mode for further handling */
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
|
|
b guest_exit_cont
|
|
|
|
/*
|
|
* Call C code to handle a HMI in real mode.
|
|
* Only the primary thread does the call, secondary threads are handled
|
|
* by calling hmi_exception_realmode() after kvmppc_hv_entry returns.
|
|
* r9 points to the vcpu on entry
|
|
*/
|
|
hmi_realmode:
|
|
lbz r0, HSTATE_PTID(r13)
|
|
cmpwi r0, 0
|
|
bne guest_exit_cont
|
|
bl kvmppc_realmode_hmi_handler
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
li r12, BOOK3S_INTERRUPT_HMI
|
|
b guest_exit_cont
|
|
|
|
/*
|
|
* Check the reason we woke from nap, and take appropriate action.
|
|
* Returns (in r3):
|
|
* 0 if nothing needs to be done
|
|
* 1 if something happened that needs to be handled by the host
|
|
* -1 if there was a guest wakeup (IPI or msgsnd)
|
|
* -2 if we handled a PCI passthrough interrupt (returned by
|
|
* kvmppc_read_intr only)
|
|
*
|
|
* Also sets r12 to the interrupt vector for any interrupt that needs
|
|
* to be handled now by the host (0x500 for external interrupt), or zero.
|
|
* Modifies all volatile registers (since it may call a C function).
|
|
* This routine calls kvmppc_read_intr, a C function, if an external
|
|
* interrupt is pending.
|
|
*/
|
|
kvmppc_check_wake_reason:
|
|
mfspr r6, SPRN_SRR1
|
|
BEGIN_FTR_SECTION
|
|
rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */
|
|
FTR_SECTION_ELSE
|
|
rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */
|
|
ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
|
|
cmpwi r6, 8 /* was it an external interrupt? */
|
|
beq 7f /* if so, see what it was */
|
|
li r3, 0
|
|
li r12, 0
|
|
cmpwi r6, 6 /* was it the decrementer? */
|
|
beq 0f
|
|
BEGIN_FTR_SECTION
|
|
cmpwi r6, 5 /* privileged doorbell? */
|
|
beq 0f
|
|
cmpwi r6, 3 /* hypervisor doorbell? */
|
|
beq 3f
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
cmpwi r6, 0xa /* Hypervisor maintenance ? */
|
|
beq 4f
|
|
li r3, 1 /* anything else, return 1 */
|
|
0: blr
|
|
|
|
/* hypervisor doorbell */
|
|
3: li r12, BOOK3S_INTERRUPT_H_DOORBELL
|
|
|
|
/*
|
|
* Clear the doorbell as we will invoke the handler
|
|
* explicitly in the guest exit path.
|
|
*/
|
|
lis r6, (PPC_DBELL_SERVER << (63-36))@h
|
|
PPC_MSGCLR(6)
|
|
/* see if it's a host IPI */
|
|
li r3, 1
|
|
lbz r0, HSTATE_HOST_IPI(r13)
|
|
cmpwi r0, 0
|
|
bnelr
|
|
/* if not, return -1 */
|
|
li r3, -1
|
|
blr
|
|
|
|
/* Woken up due to Hypervisor maintenance interrupt */
|
|
4: li r12, BOOK3S_INTERRUPT_HMI
|
|
li r3, 1
|
|
blr
|
|
|
|
/* external interrupt - create a stack frame so we can call C */
|
|
7: mflr r0
|
|
std r0, PPC_LR_STKOFF(r1)
|
|
stdu r1, -PPC_MIN_STKFRM(r1)
|
|
bl kvmppc_read_intr
|
|
nop
|
|
li r12, BOOK3S_INTERRUPT_EXTERNAL
|
|
cmpdi r3, 1
|
|
ble 1f
|
|
|
|
/*
|
|
* Return code of 2 means PCI passthrough interrupt, but
|
|
* we need to return back to host to complete handling the
|
|
* interrupt. Trap reason is expected in r12 by guest
|
|
* exit code.
|
|
*/
|
|
li r12, BOOK3S_INTERRUPT_HV_RM_HARD
|
|
1:
|
|
ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1)
|
|
addi r1, r1, PPC_MIN_STKFRM
|
|
mtlr r0
|
|
blr
|
|
|
|
/*
|
|
* Save away FP, VMX and VSX registers.
|
|
* r3 = vcpu pointer
|
|
* N.B. r30 and r31 are volatile across this function,
|
|
* thus it is not callable from C.
|
|
*/
|
|
kvmppc_save_fp:
|
|
mflr r30
|
|
mr r31,r3
|
|
mfmsr r5
|
|
ori r8,r5,MSR_FP
|
|
#ifdef CONFIG_ALTIVEC
|
|
BEGIN_FTR_SECTION
|
|
oris r8,r8,MSR_VEC@h
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
#endif
|
|
#ifdef CONFIG_VSX
|
|
BEGIN_FTR_SECTION
|
|
oris r8,r8,MSR_VSX@h
|
|
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
|
|
#endif
|
|
mtmsrd r8
|
|
addi r3,r3,VCPU_FPRS
|
|
bl store_fp_state
|
|
#ifdef CONFIG_ALTIVEC
|
|
BEGIN_FTR_SECTION
|
|
addi r3,r31,VCPU_VRS
|
|
bl store_vr_state
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
#endif
|
|
mfspr r6,SPRN_VRSAVE
|
|
stw r6,VCPU_VRSAVE(r31)
|
|
mtlr r30
|
|
blr
|
|
|
|
/*
|
|
* Load up FP, VMX and VSX registers
|
|
* r4 = vcpu pointer
|
|
* N.B. r30 and r31 are volatile across this function,
|
|
* thus it is not callable from C.
|
|
*/
|
|
kvmppc_load_fp:
|
|
mflr r30
|
|
mr r31,r4
|
|
mfmsr r9
|
|
ori r8,r9,MSR_FP
|
|
#ifdef CONFIG_ALTIVEC
|
|
BEGIN_FTR_SECTION
|
|
oris r8,r8,MSR_VEC@h
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
#endif
|
|
#ifdef CONFIG_VSX
|
|
BEGIN_FTR_SECTION
|
|
oris r8,r8,MSR_VSX@h
|
|
END_FTR_SECTION_IFSET(CPU_FTR_VSX)
|
|
#endif
|
|
mtmsrd r8
|
|
addi r3,r4,VCPU_FPRS
|
|
bl load_fp_state
|
|
#ifdef CONFIG_ALTIVEC
|
|
BEGIN_FTR_SECTION
|
|
addi r3,r31,VCPU_VRS
|
|
bl load_vr_state
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
|
|
#endif
|
|
lwz r7,VCPU_VRSAVE(r31)
|
|
mtspr SPRN_VRSAVE,r7
|
|
mtlr r30
|
|
mr r4,r31
|
|
blr
|
|
|
|
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
|
|
/*
|
|
* Save transactional state and TM-related registers.
|
|
* Called with r3 pointing to the vcpu struct and r4 containing
|
|
* the guest MSR value.
|
|
* r5 is non-zero iff non-volatile register state needs to be maintained.
|
|
* If r5 == 0, this can modify all checkpointed registers, but
|
|
* restores r1 and r2 before exit.
|
|
*/
|
|
_GLOBAL_TOC(kvmppc_save_tm_hv)
|
|
EXPORT_SYMBOL_GPL(kvmppc_save_tm_hv)
|
|
/* See if we need to handle fake suspend mode */
|
|
BEGIN_FTR_SECTION
|
|
b __kvmppc_save_tm
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
|
|
|
|
lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */
|
|
cmpwi r0, 0
|
|
beq __kvmppc_save_tm
|
|
|
|
/* The following code handles the fake_suspend = 1 case */
|
|
mflr r0
|
|
std r0, PPC_LR_STKOFF(r1)
|
|
stdu r1, -TM_FRAME_SIZE(r1)
|
|
|
|
/* Turn on TM. */
|
|
mfmsr r8
|
|
li r0, 1
|
|
rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
|
|
mtmsrd r8
|
|
|
|
rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */
|
|
beq 4f
|
|
BEGIN_FTR_SECTION
|
|
bl pnv_power9_force_smt4_catch
|
|
END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
|
|
nop
|
|
|
|
/*
|
|
* It's possible that treclaim. may modify registers, if we have lost
|
|
* track of fake-suspend state in the guest due to it using rfscv.
|
|
* Save and restore registers in case this occurs.
|
|
*/
|
|
mfspr r3, SPRN_DSCR
|
|
mfspr r4, SPRN_XER
|
|
mfspr r5, SPRN_AMR
|
|
/* SPRN_TAR would need to be saved here if the kernel ever used it */
|
|
mfcr r12
|
|
SAVE_NVGPRS(r1)
|
|
SAVE_GPR(2, r1)
|
|
SAVE_GPR(3, r1)
|
|
SAVE_GPR(4, r1)
|
|
SAVE_GPR(5, r1)
|
|
stw r12, 8(r1)
|
|
std r1, HSTATE_HOST_R1(r13)
|
|
|
|
/* We have to treclaim here because that's the only way to do S->N */
|
|
li r3, TM_CAUSE_KVM_RESCHED
|
|
TRECLAIM(R3)
|
|
|
|
GET_PACA(r13)
|
|
ld r1, HSTATE_HOST_R1(r13)
|
|
REST_GPR(2, r1)
|
|
REST_GPR(3, r1)
|
|
REST_GPR(4, r1)
|
|
REST_GPR(5, r1)
|
|
lwz r12, 8(r1)
|
|
REST_NVGPRS(r1)
|
|
mtspr SPRN_DSCR, r3
|
|
mtspr SPRN_XER, r4
|
|
mtspr SPRN_AMR, r5
|
|
mtcr r12
|
|
HMT_MEDIUM
|
|
|
|
/*
|
|
* We were in fake suspend, so we are not going to save the
|
|
* register state as the guest checkpointed state (since
|
|
* we already have it), therefore we can now use any volatile GPR.
|
|
* In fact treclaim in fake suspend state doesn't modify
|
|
* any registers.
|
|
*/
|
|
|
|
BEGIN_FTR_SECTION
|
|
bl pnv_power9_force_smt4_release
|
|
END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
|
|
nop
|
|
|
|
4:
|
|
mfspr r3, SPRN_PSSCR
|
|
/* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */
|
|
li r0, PSSCR_FAKE_SUSPEND
|
|
andc r3, r3, r0
|
|
mtspr SPRN_PSSCR, r3
|
|
|
|
/* Don't save TEXASR, use value from last exit in real suspend state */
|
|
ld r9, HSTATE_KVM_VCPU(r13)
|
|
mfspr r5, SPRN_TFHAR
|
|
mfspr r6, SPRN_TFIAR
|
|
std r5, VCPU_TFHAR(r9)
|
|
std r6, VCPU_TFIAR(r9)
|
|
|
|
addi r1, r1, TM_FRAME_SIZE
|
|
ld r0, PPC_LR_STKOFF(r1)
|
|
mtlr r0
|
|
blr
|
|
|
|
/*
|
|
* Restore transactional state and TM-related registers.
|
|
* Called with r3 pointing to the vcpu struct
|
|
* and r4 containing the guest MSR value.
|
|
* r5 is non-zero iff non-volatile register state needs to be maintained.
|
|
* This potentially modifies all checkpointed registers.
|
|
* It restores r1 and r2 from the PACA.
|
|
*/
|
|
_GLOBAL_TOC(kvmppc_restore_tm_hv)
|
|
EXPORT_SYMBOL_GPL(kvmppc_restore_tm_hv)
|
|
/*
|
|
* If we are doing TM emulation for the guest on a POWER9 DD2,
|
|
* then we don't actually do a trechkpt -- we either set up
|
|
* fake-suspend mode, or emulate a TM rollback.
|
|
*/
|
|
BEGIN_FTR_SECTION
|
|
b __kvmppc_restore_tm
|
|
END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
|
|
mflr r0
|
|
std r0, PPC_LR_STKOFF(r1)
|
|
|
|
li r0, 0
|
|
stb r0, HSTATE_FAKE_SUSPEND(r13)
|
|
|
|
/* Turn on TM so we can restore TM SPRs */
|
|
mfmsr r5
|
|
li r0, 1
|
|
rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG
|
|
mtmsrd r5
|
|
|
|
/*
|
|
* The user may change these outside of a transaction, so they must
|
|
* always be context switched.
|
|
*/
|
|
ld r5, VCPU_TFHAR(r3)
|
|
ld r6, VCPU_TFIAR(r3)
|
|
ld r7, VCPU_TEXASR(r3)
|
|
mtspr SPRN_TFHAR, r5
|
|
mtspr SPRN_TFIAR, r6
|
|
mtspr SPRN_TEXASR, r7
|
|
|
|
rldicl. r5, r4, 64 - MSR_TS_S_LG, 62
|
|
beqlr /* TM not active in guest */
|
|
|
|
/* Make sure the failure summary is set */
|
|
oris r7, r7, (TEXASR_FS)@h
|
|
mtspr SPRN_TEXASR, r7
|
|
|
|
cmpwi r5, 1 /* check for suspended state */
|
|
bgt 10f
|
|
stb r5, HSTATE_FAKE_SUSPEND(r13)
|
|
b 9f /* and return */
|
|
10: stdu r1, -PPC_MIN_STKFRM(r1)
|
|
/* guest is in transactional state, so simulate rollback */
|
|
bl kvmhv_emulate_tm_rollback
|
|
nop
|
|
addi r1, r1, PPC_MIN_STKFRM
|
|
9: ld r0, PPC_LR_STKOFF(r1)
|
|
mtlr r0
|
|
blr
|
|
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
|
|
|
|
/*
|
|
* We come here if we get any exception or interrupt while we are
|
|
* executing host real mode code while in guest MMU context.
|
|
* r12 is (CR << 32) | vector
|
|
* r13 points to our PACA
|
|
* r12 is saved in HSTATE_SCRATCH0(r13)
|
|
* r9 is saved in HSTATE_SCRATCH2(r13)
|
|
* r13 is saved in HSPRG1
|
|
* cfar is saved in HSTATE_CFAR(r13)
|
|
* ppr is saved in HSTATE_PPR(r13)
|
|
*/
|
|
kvmppc_bad_host_intr:
|
|
/*
|
|
* Switch to the emergency stack, but start half-way down in
|
|
* case we were already on it.
|
|
*/
|
|
mr r9, r1
|
|
std r1, PACAR1(r13)
|
|
ld r1, PACAEMERGSP(r13)
|
|
subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE
|
|
std r9, 0(r1)
|
|
std r0, GPR0(r1)
|
|
std r9, GPR1(r1)
|
|
std r2, GPR2(r1)
|
|
SAVE_4GPRS(3, r1)
|
|
SAVE_2GPRS(7, r1)
|
|
srdi r0, r12, 32
|
|
clrldi r12, r12, 32
|
|
std r0, _CCR(r1)
|
|
std r12, _TRAP(r1)
|
|
andi. r0, r12, 2
|
|
beq 1f
|
|
mfspr r3, SPRN_HSRR0
|
|
mfspr r4, SPRN_HSRR1
|
|
mfspr r5, SPRN_HDAR
|
|
mfspr r6, SPRN_HDSISR
|
|
b 2f
|
|
1: mfspr r3, SPRN_SRR0
|
|
mfspr r4, SPRN_SRR1
|
|
mfspr r5, SPRN_DAR
|
|
mfspr r6, SPRN_DSISR
|
|
2: std r3, _NIP(r1)
|
|
std r4, _MSR(r1)
|
|
std r5, _DAR(r1)
|
|
std r6, _DSISR(r1)
|
|
ld r9, HSTATE_SCRATCH2(r13)
|
|
ld r12, HSTATE_SCRATCH0(r13)
|
|
GET_SCRATCH0(r0)
|
|
SAVE_4GPRS(9, r1)
|
|
std r0, GPR13(r1)
|
|
SAVE_NVGPRS(r1)
|
|
ld r5, HSTATE_CFAR(r13)
|
|
std r5, ORIG_GPR3(r1)
|
|
mflr r3
|
|
mfctr r4
|
|
mfxer r5
|
|
lbz r6, PACAIRQSOFTMASK(r13)
|
|
std r3, _LINK(r1)
|
|
std r4, _CTR(r1)
|
|
std r5, _XER(r1)
|
|
std r6, SOFTE(r1)
|
|
ld r2, PACATOC(r13)
|
|
LOAD_REG_IMMEDIATE(3, 0x7265677368657265)
|
|
std r3, STACK_FRAME_OVERHEAD-16(r1)
|
|
|
|
/*
|
|
* XXX On POWER7 and POWER8, we just spin here since we don't
|
|
* know what the other threads are doing (and we don't want to
|
|
* coordinate with them) - but at least we now have register state
|
|
* in memory that we might be able to look at from another CPU.
|
|
*/
|
|
b .
|
|
|
|
/*
|
|
* This mimics the MSR transition on IRQ delivery. The new guest MSR is taken
|
|
* from VCPU_INTR_MSR and is modified based on the required TM state changes.
|
|
* r11 has the guest MSR value (in/out)
|
|
* r9 has a vcpu pointer (in)
|
|
* r0 is used as a scratch register
|
|
*/
|
|
kvmppc_msr_interrupt:
|
|
rldicl r0, r11, 64 - MSR_TS_S_LG, 62
|
|
cmpwi r0, 2 /* Check if we are in transactional state.. */
|
|
ld r11, VCPU_INTR_MSR(r9)
|
|
bne 1f
|
|
/* ... if transactional, change to suspended */
|
|
li r0, 1
|
|
1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
|
|
blr
|
|
|
|
/*
|
|
* Load up guest PMU state. R3 points to the vcpu struct.
|
|
*/
|
|
_GLOBAL(kvmhv_load_guest_pmu)
|
|
EXPORT_SYMBOL_GPL(kvmhv_load_guest_pmu)
|
|
mr r4, r3
|
|
mflr r0
|
|
li r3, 1
|
|
sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
|
|
mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
|
|
isync
|
|
BEGIN_FTR_SECTION
|
|
ld r3, VCPU_MMCR(r4)
|
|
andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
|
|
cmpwi r5, MMCR0_PMAO
|
|
beql kvmppc_fix_pmao
|
|
END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
|
|
lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
|
|
lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
|
|
lwz r6, VCPU_PMC + 8(r4)
|
|
lwz r7, VCPU_PMC + 12(r4)
|
|
lwz r8, VCPU_PMC + 16(r4)
|
|
lwz r9, VCPU_PMC + 20(r4)
|
|
mtspr SPRN_PMC1, r3
|
|
mtspr SPRN_PMC2, r5
|
|
mtspr SPRN_PMC3, r6
|
|
mtspr SPRN_PMC4, r7
|
|
mtspr SPRN_PMC5, r8
|
|
mtspr SPRN_PMC6, r9
|
|
ld r3, VCPU_MMCR(r4)
|
|
ld r5, VCPU_MMCR + 8(r4)
|
|
ld r6, VCPU_MMCRA(r4)
|
|
ld r7, VCPU_SIAR(r4)
|
|
ld r8, VCPU_SDAR(r4)
|
|
mtspr SPRN_MMCR1, r5
|
|
mtspr SPRN_MMCRA, r6
|
|
mtspr SPRN_SIAR, r7
|
|
mtspr SPRN_SDAR, r8
|
|
BEGIN_FTR_SECTION
|
|
ld r5, VCPU_MMCR + 24(r4)
|
|
ld r6, VCPU_SIER + 8(r4)
|
|
ld r7, VCPU_SIER + 16(r4)
|
|
mtspr SPRN_MMCR3, r5
|
|
mtspr SPRN_SIER2, r6
|
|
mtspr SPRN_SIER3, r7
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31)
|
|
BEGIN_FTR_SECTION
|
|
ld r5, VCPU_MMCR + 16(r4)
|
|
ld r6, VCPU_SIER(r4)
|
|
mtspr SPRN_MMCR2, r5
|
|
mtspr SPRN_SIER, r6
|
|
BEGIN_FTR_SECTION_NESTED(96)
|
|
lwz r7, VCPU_PMC + 24(r4)
|
|
lwz r8, VCPU_PMC + 28(r4)
|
|
ld r9, VCPU_MMCRS(r4)
|
|
mtspr SPRN_SPMC1, r7
|
|
mtspr SPRN_SPMC2, r8
|
|
mtspr SPRN_MMCRS, r9
|
|
END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
mtspr SPRN_MMCR0, r3
|
|
isync
|
|
mtlr r0
|
|
blr
|
|
|
|
/*
|
|
* Reload host PMU state saved in the PACA by kvmhv_save_host_pmu.
|
|
*/
|
|
_GLOBAL(kvmhv_load_host_pmu)
|
|
EXPORT_SYMBOL_GPL(kvmhv_load_host_pmu)
|
|
mflr r0
|
|
lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */
|
|
cmpwi r4, 0
|
|
beq 23f /* skip if not */
|
|
BEGIN_FTR_SECTION
|
|
ld r3, HSTATE_MMCR0(r13)
|
|
andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
|
|
cmpwi r4, MMCR0_PMAO
|
|
beql kvmppc_fix_pmao
|
|
END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
|
|
lwz r3, HSTATE_PMC1(r13)
|
|
lwz r4, HSTATE_PMC2(r13)
|
|
lwz r5, HSTATE_PMC3(r13)
|
|
lwz r6, HSTATE_PMC4(r13)
|
|
lwz r8, HSTATE_PMC5(r13)
|
|
lwz r9, HSTATE_PMC6(r13)
|
|
mtspr SPRN_PMC1, r3
|
|
mtspr SPRN_PMC2, r4
|
|
mtspr SPRN_PMC3, r5
|
|
mtspr SPRN_PMC4, r6
|
|
mtspr SPRN_PMC5, r8
|
|
mtspr SPRN_PMC6, r9
|
|
ld r3, HSTATE_MMCR0(r13)
|
|
ld r4, HSTATE_MMCR1(r13)
|
|
ld r5, HSTATE_MMCRA(r13)
|
|
ld r6, HSTATE_SIAR(r13)
|
|
ld r7, HSTATE_SDAR(r13)
|
|
mtspr SPRN_MMCR1, r4
|
|
mtspr SPRN_MMCRA, r5
|
|
mtspr SPRN_SIAR, r6
|
|
mtspr SPRN_SDAR, r7
|
|
BEGIN_FTR_SECTION
|
|
ld r8, HSTATE_MMCR2(r13)
|
|
ld r9, HSTATE_SIER(r13)
|
|
mtspr SPRN_MMCR2, r8
|
|
mtspr SPRN_SIER, r9
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
BEGIN_FTR_SECTION
|
|
ld r5, HSTATE_MMCR3(r13)
|
|
ld r6, HSTATE_SIER2(r13)
|
|
ld r7, HSTATE_SIER3(r13)
|
|
mtspr SPRN_MMCR3, r5
|
|
mtspr SPRN_SIER2, r6
|
|
mtspr SPRN_SIER3, r7
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31)
|
|
mtspr SPRN_MMCR0, r3
|
|
isync
|
|
mtlr r0
|
|
23: blr
|
|
|
|
/*
|
|
* Save guest PMU state into the vcpu struct.
|
|
* r3 = vcpu, r4 = full save flag (PMU in use flag set in VPA)
|
|
*/
|
|
_GLOBAL(kvmhv_save_guest_pmu)
|
|
EXPORT_SYMBOL_GPL(kvmhv_save_guest_pmu)
|
|
mr r9, r3
|
|
mr r8, r4
|
|
BEGIN_FTR_SECTION
|
|
/*
|
|
* POWER8 seems to have a hardware bug where setting
|
|
* MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
|
|
* when some counters are already negative doesn't seem
|
|
* to cause a performance monitor alert (and hence interrupt).
|
|
* The effect of this is that when saving the PMU state,
|
|
* if there is no PMU alert pending when we read MMCR0
|
|
* before freezing the counters, but one becomes pending
|
|
* before we read the counters, we lose it.
|
|
* To work around this, we need a way to freeze the counters
|
|
* before reading MMCR0. Normally, freezing the counters
|
|
* is done by writing MMCR0 (to set MMCR0[FC]) which
|
|
* unavoidably writes MMCR0[PMA0] as well. On POWER8,
|
|
* we can also freeze the counters using MMCR2, by writing
|
|
* 1s to all the counter freeze condition bits (there are
|
|
* 9 bits each for 6 counters).
|
|
*/
|
|
li r3, -1 /* set all freeze bits */
|
|
clrrdi r3, r3, 10
|
|
mfspr r10, SPRN_MMCR2
|
|
mtspr SPRN_MMCR2, r3
|
|
isync
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
li r3, 1
|
|
sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
|
|
mfspr r4, SPRN_MMCR0 /* save MMCR0 */
|
|
mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
|
|
mfspr r6, SPRN_MMCRA
|
|
/* Clear MMCRA in order to disable SDAR updates */
|
|
li r7, 0
|
|
mtspr SPRN_MMCRA, r7
|
|
isync
|
|
cmpwi r8, 0 /* did they ask for PMU stuff to be saved? */
|
|
bne 21f
|
|
std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
|
|
b 22f
|
|
21: mfspr r5, SPRN_MMCR1
|
|
mfspr r7, SPRN_SIAR
|
|
mfspr r8, SPRN_SDAR
|
|
std r4, VCPU_MMCR(r9)
|
|
std r5, VCPU_MMCR + 8(r9)
|
|
std r6, VCPU_MMCRA(r9)
|
|
BEGIN_FTR_SECTION
|
|
std r10, VCPU_MMCR + 16(r9)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
BEGIN_FTR_SECTION
|
|
mfspr r5, SPRN_MMCR3
|
|
mfspr r6, SPRN_SIER2
|
|
mfspr r7, SPRN_SIER3
|
|
std r5, VCPU_MMCR + 24(r9)
|
|
std r6, VCPU_SIER + 8(r9)
|
|
std r7, VCPU_SIER + 16(r9)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31)
|
|
std r7, VCPU_SIAR(r9)
|
|
std r8, VCPU_SDAR(r9)
|
|
mfspr r3, SPRN_PMC1
|
|
mfspr r4, SPRN_PMC2
|
|
mfspr r5, SPRN_PMC3
|
|
mfspr r6, SPRN_PMC4
|
|
mfspr r7, SPRN_PMC5
|
|
mfspr r8, SPRN_PMC6
|
|
stw r3, VCPU_PMC(r9)
|
|
stw r4, VCPU_PMC + 4(r9)
|
|
stw r5, VCPU_PMC + 8(r9)
|
|
stw r6, VCPU_PMC + 12(r9)
|
|
stw r7, VCPU_PMC + 16(r9)
|
|
stw r8, VCPU_PMC + 20(r9)
|
|
BEGIN_FTR_SECTION
|
|
mfspr r5, SPRN_SIER
|
|
std r5, VCPU_SIER(r9)
|
|
BEGIN_FTR_SECTION_NESTED(96)
|
|
mfspr r6, SPRN_SPMC1
|
|
mfspr r7, SPRN_SPMC2
|
|
mfspr r8, SPRN_MMCRS
|
|
stw r6, VCPU_PMC + 24(r9)
|
|
stw r7, VCPU_PMC + 28(r9)
|
|
std r8, VCPU_MMCRS(r9)
|
|
lis r4, 0x8000
|
|
mtspr SPRN_MMCRS, r4
|
|
END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96)
|
|
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
|
|
22: blr
|
|
|
|
/*
|
|
* This works around a hardware bug on POWER8E processors, where
|
|
* writing a 1 to the MMCR0[PMAO] bit doesn't generate a
|
|
* performance monitor interrupt. Instead, when we need to have
|
|
* an interrupt pending, we have to arrange for a counter to overflow.
|
|
*/
|
|
kvmppc_fix_pmao:
|
|
li r3, 0
|
|
mtspr SPRN_MMCR2, r3
|
|
lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
|
|
ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
|
|
mtspr SPRN_MMCR0, r3
|
|
lis r3, 0x7fff
|
|
ori r3, r3, 0xffff
|
|
mtspr SPRN_PMC6, r3
|
|
isync
|
|
blr
|
|
|
|
#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
|
|
/*
|
|
* Start timing an activity
|
|
* r3 = pointer to time accumulation struct, r4 = vcpu
|
|
*/
|
|
kvmhv_start_timing:
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
ld r6, VCORE_TB_OFFSET_APPL(r5)
|
|
mftb r5
|
|
subf r5, r6, r5 /* subtract current timebase offset */
|
|
std r3, VCPU_CUR_ACTIVITY(r4)
|
|
std r5, VCPU_ACTIVITY_START(r4)
|
|
blr
|
|
|
|
/*
|
|
* Accumulate time to one activity and start another.
|
|
* r3 = pointer to new time accumulation struct, r4 = vcpu
|
|
*/
|
|
kvmhv_accumulate_time:
|
|
ld r5, HSTATE_KVM_VCORE(r13)
|
|
ld r8, VCORE_TB_OFFSET_APPL(r5)
|
|
ld r5, VCPU_CUR_ACTIVITY(r4)
|
|
ld r6, VCPU_ACTIVITY_START(r4)
|
|
std r3, VCPU_CUR_ACTIVITY(r4)
|
|
mftb r7
|
|
subf r7, r8, r7 /* subtract current timebase offset */
|
|
std r7, VCPU_ACTIVITY_START(r4)
|
|
cmpdi r5, 0
|
|
beqlr
|
|
subf r3, r6, r7
|
|
ld r8, TAS_SEQCOUNT(r5)
|
|
cmpdi r8, 0
|
|
addi r8, r8, 1
|
|
std r8, TAS_SEQCOUNT(r5)
|
|
lwsync
|
|
ld r7, TAS_TOTAL(r5)
|
|
add r7, r7, r3
|
|
std r7, TAS_TOTAL(r5)
|
|
ld r6, TAS_MIN(r5)
|
|
ld r7, TAS_MAX(r5)
|
|
beq 3f
|
|
cmpd r3, r6
|
|
bge 1f
|
|
3: std r3, TAS_MIN(r5)
|
|
1: cmpd r3, r7
|
|
ble 2f
|
|
std r3, TAS_MAX(r5)
|
|
2: lwsync
|
|
addi r8, r8, 1
|
|
std r8, TAS_SEQCOUNT(r5)
|
|
blr
|
|
#endif
|