korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-12-03 00:54:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

KVM: PPC: Book3S: Invalidate multiple TCEs at once

Browse Source 
Invalidating a TCE cache entry for each updated TCE is quite expensive.
This makes use of the new iommu_table_ops::xchg_no_kill()/tce_kill()
callbacks to bring down the time spent in mapping a huge guest DMA window;
roughly 20s to 10s for each guest's 100GB of DMA space.

Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Acked-by: Paul Mackerras <paulus@ozlabs.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20190829085252.72370-3-aik@ozlabs.ru
This commit is contained in:
Alexey Kardashevskiy 2019-08-29 18:52:49 +10:00 committed by Michael Ellerman
parent 35872480da
commit 01b7d128b5
2 changed files with 48 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
arch/powerpc/kvm/book3s_64_vio.c
View File

@ -416,7 +416,7 @@ static void kvmppc_clear_tce(struct mm_struct *mm, struct iommu_table *tbl,
unsigned long hpa = 0; unsigned long hpa = 0;
enum dma_data_direction dir = DMA_NONE; enum dma_data_direction dir = DMA_NONE;
iommu_tce_xchg(mm, tbl, entry, &hpa, &dir); iommu_tce_xchg_no_kill(mm, tbl, entry, &hpa, &dir);
} }
static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm, static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm,
@ -447,7 +447,8 @@ static long kvmppc_tce_iommu_do_unmap(struct kvm *kvm,
unsigned long hpa = 0; unsigned long hpa = 0;
long ret; long ret;
if (WARN_ON_ONCE(iommu_tce_xchg(kvm->mm, tbl, entry, &hpa, &dir))) if (WARN_ON_ONCE(iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa,
&dir)))
return H_TOO_HARD; return H_TOO_HARD;
if (dir == DMA_NONE) if (dir == DMA_NONE)
@ -455,7 +456,7 @@ static long kvmppc_tce_iommu_do_unmap(struct kvm *kvm,
ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry); ret = kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry);
if (ret != H_SUCCESS) if (ret != H_SUCCESS)
iommu_tce_xchg(kvm->mm, tbl, entry, &hpa, &dir); iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);
return ret; return ret;
} }
@ -501,7 +502,7 @@ long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
if (mm_iommu_mapped_inc(mem)) if (mm_iommu_mapped_inc(mem))
return H_TOO_HARD; return H_TOO_HARD;
ret = iommu_tce_xchg(kvm->mm, tbl, entry, &hpa, &dir); ret = iommu_tce_xchg_no_kill(kvm->mm, tbl, entry, &hpa, &dir);
if (WARN_ON_ONCE(ret)) { if (WARN_ON_ONCE(ret)) {
mm_iommu_mapped_dec(mem); mm_iommu_mapped_dec(mem);
return H_TOO_HARD; return H_TOO_HARD;
@ -579,6 +580,8 @@ long kvmppc_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl, ret = kvmppc_tce_iommu_map(vcpu->kvm, stt, stit->tbl,
entry, ua, dir); entry, ua, dir);
iommu_tce_kill(stit->tbl, entry, 1);
if (ret != H_SUCCESS) { if (ret != H_SUCCESS) {
kvmppc_clear_tce(vcpu->kvm->mm, stit->tbl, entry); kvmppc_clear_tce(vcpu->kvm->mm, stit->tbl, entry);
goto unlock_exit; goto unlock_exit;
@ -656,13 +659,13 @@ long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
*/ */
if (get_user(tce, tces + i)) { if (get_user(tce, tces + i)) {
ret = H_TOO_HARD; ret = H_TOO_HARD;
goto unlock_exit; goto invalidate_exit;
} }
tce = be64_to_cpu(tce); tce = be64_to_cpu(tce);
if (kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) { if (kvmppc_tce_to_ua(vcpu->kvm, tce, &ua)) {
ret = H_PARAMETER; ret = H_PARAMETER;
goto unlock_exit; goto invalidate_exit;
} }
list_for_each_entry_lockless(stit, &stt->iommu_tables, next) { list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
@ -673,13 +676,17 @@ long kvmppc_h_put_tce_indirect(struct kvm_vcpu *vcpu,
if (ret != H_SUCCESS) { if (ret != H_SUCCESS) {
kvmppc_clear_tce(vcpu->kvm->mm, stit->tbl, kvmppc_clear_tce(vcpu->kvm->mm, stit->tbl,
entry); entry);
goto unlock_exit; goto invalidate_exit;
} }
} }
kvmppc_tce_put(stt, entry + i, tce); kvmppc_tce_put(stt, entry + i, tce);
} }
invalidate_exit:
list_for_each_entry_lockless(stit, &stt->iommu_tables, next)
iommu_tce_kill(stit->tbl, entry, npages);
unlock_exit: unlock_exit:
srcu_read_unlock(&vcpu->kvm->srcu, idx); srcu_read_unlock(&vcpu->kvm->srcu, idx);
@ -718,7 +725,7 @@ long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu,
continue; continue;
if (ret == H_TOO_HARD) if (ret == H_TOO_HARD)
return ret; goto invalidate_exit;
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
kvmppc_clear_tce(vcpu->kvm->mm, stit->tbl, entry); kvmppc_clear_tce(vcpu->kvm->mm, stit->tbl, entry);
@ -728,6 +735,10 @@ long kvmppc_h_stuff_tce(struct kvm_vcpu *vcpu,
for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift)) for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))
kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value); kvmppc_tce_put(stt, ioba >> stt->page_shift, tce_value);
return H_SUCCESS; invalidate_exit:
list_for_each_entry_lockless(stit, &stt->iommu_tables, next)
iommu_tce_kill(stit->tbl, ioba >> stt->page_shift, npages);
return ret;
} }
EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce); EXPORT_SYMBOL_GPL(kvmppc_h_stuff_tce);

38
arch/powerpc/kvm/book3s_64_vio_hv.c
View File

@ -218,13 +218,14 @@ static long kvmppc_rm_ioba_validate(struct kvmppc_spapr_tce_table *stt,
return H_SUCCESS; return H_SUCCESS;
} }
static long iommu_tce_xchg_rm(struct mm_struct *mm, struct iommu_table *tbl, static long iommu_tce_xchg_no_kill_rm(struct mm_struct *mm,
struct iommu_table *tbl,
unsigned long entry, unsigned long *hpa, unsigned long entry, unsigned long *hpa,
enum dma_data_direction *direction) enum dma_data_direction *direction)
{ {
long ret; long ret;
ret = tbl->it_ops->exchange_rm(tbl, entry, hpa, direction); ret = tbl->it_ops->xchg_no_kill(tbl, entry, hpa, direction, true);
if (!ret && ((*direction == DMA_FROM_DEVICE) || if (!ret && ((*direction == DMA_FROM_DEVICE) ||
(*direction == DMA_BIDIRECTIONAL))) { (*direction == DMA_BIDIRECTIONAL))) {
@ -240,13 +241,20 @@ static long iommu_tce_xchg_rm(struct mm_struct *mm, struct iommu_table *tbl,
return ret; return ret;
} }
extern void iommu_tce_kill_rm(struct iommu_table *tbl,
unsigned long entry, unsigned long pages)
{
if (tbl->it_ops->tce_kill)
tbl->it_ops->tce_kill(tbl, entry, pages, true);
}
static void kvmppc_rm_clear_tce(struct kvm *kvm, struct iommu_table *tbl, static void kvmppc_rm_clear_tce(struct kvm *kvm, struct iommu_table *tbl,
unsigned long entry) unsigned long entry)
{ {
unsigned long hpa = 0; unsigned long hpa = 0;
enum dma_data_direction dir = DMA_NONE; enum dma_data_direction dir = DMA_NONE;
iommu_tce_xchg_rm(kvm->mm, tbl, entry, &hpa, &dir); iommu_tce_xchg_no_kill_rm(kvm->mm, tbl, entry, &hpa, &dir);
} }
static long kvmppc_rm_tce_iommu_mapped_dec(struct kvm *kvm, static long kvmppc_rm_tce_iommu_mapped_dec(struct kvm *kvm,
@ -278,7 +286,7 @@ static long kvmppc_rm_tce_iommu_do_unmap(struct kvm *kvm,
unsigned long hpa = 0; unsigned long hpa = 0;
long ret; long ret;
if (iommu_tce_xchg_rm(kvm->mm, tbl, entry, &hpa, &dir)) if (iommu_tce_xchg_no_kill_rm(kvm->mm, tbl, entry, &hpa, &dir))
/* /*
* real mode xchg can fail if struct page crosses * real mode xchg can fail if struct page crosses
* a page boundary * a page boundary
@ -290,7 +298,7 @@ static long kvmppc_rm_tce_iommu_do_unmap(struct kvm *kvm,
ret = kvmppc_rm_tce_iommu_mapped_dec(kvm, tbl, entry); ret = kvmppc_rm_tce_iommu_mapped_dec(kvm, tbl, entry);
if (ret) if (ret)
iommu_tce_xchg_rm(kvm->mm, tbl, entry, &hpa, &dir); iommu_tce_xchg_no_kill_rm(kvm->mm, tbl, entry, &hpa, &dir);
return ret; return ret;
} }
@ -336,7 +344,7 @@ static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl,
if (WARN_ON_ONCE_RM(mm_iommu_mapped_inc(mem))) if (WARN_ON_ONCE_RM(mm_iommu_mapped_inc(mem)))
return H_TOO_HARD; return H_TOO_HARD;
ret = iommu_tce_xchg_rm(kvm->mm, tbl, entry, &hpa, &dir); ret = iommu_tce_xchg_no_kill_rm(kvm->mm, tbl, entry, &hpa, &dir);
if (ret) { if (ret) {
mm_iommu_mapped_dec(mem); mm_iommu_mapped_dec(mem);
/* /*
@ -417,6 +425,8 @@ long kvmppc_rm_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn,
ret = kvmppc_rm_tce_iommu_map(vcpu->kvm, stt, ret = kvmppc_rm_tce_iommu_map(vcpu->kvm, stt,
stit->tbl, entry, ua, dir); stit->tbl, entry, ua, dir);
iommu_tce_kill_rm(stit->tbl, entry, 1);
if (ret != H_SUCCESS) { if (ret != H_SUCCESS) {
kvmppc_rm_clear_tce(vcpu->kvm, stit->tbl, entry); kvmppc_rm_clear_tce(vcpu->kvm, stit->tbl, entry);
return ret; return ret;
@ -558,7 +568,7 @@ long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu,
ua = 0; ua = 0;
if (kvmppc_rm_tce_to_ua(vcpu->kvm, tce, &ua, NULL)) { if (kvmppc_rm_tce_to_ua(vcpu->kvm, tce, &ua, NULL)) {
ret = H_PARAMETER; ret = H_PARAMETER;
goto unlock_exit; goto invalidate_exit;
} }
list_for_each_entry_lockless(stit, &stt->iommu_tables, next) { list_for_each_entry_lockless(stit, &stt->iommu_tables, next) {
@ -569,13 +579,17 @@ long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu,
if (ret != H_SUCCESS) { if (ret != H_SUCCESS) {
kvmppc_rm_clear_tce(vcpu->kvm, stit->tbl, kvmppc_rm_clear_tce(vcpu->kvm, stit->tbl,
entry); entry);
goto unlock_exit; goto invalidate_exit;
} }
} }
kvmppc_rm_tce_put(stt, entry + i, tce); kvmppc_rm_tce_put(stt, entry + i, tce);
} }
invalidate_exit:
list_for_each_entry_lockless(stit, &stt->iommu_tables, next)
iommu_tce_kill_rm(stit->tbl, entry, npages);
unlock_exit: unlock_exit:
if (rmap) if (rmap)
unlock_rmap(rmap); unlock_rmap(rmap);
@ -618,7 +632,7 @@ long kvmppc_rm_h_stuff_tce(struct kvm_vcpu *vcpu,
continue; continue;
if (ret == H_TOO_HARD) if (ret == H_TOO_HARD)
return ret; goto invalidate_exit;
WARN_ON_ONCE_RM(1); WARN_ON_ONCE_RM(1);
kvmppc_rm_clear_tce(vcpu->kvm, stit->tbl, entry); kvmppc_rm_clear_tce(vcpu->kvm, stit->tbl, entry);
@ -628,7 +642,11 @@ long kvmppc_rm_h_stuff_tce(struct kvm_vcpu *vcpu,
for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift)) for (i = 0; i < npages; ++i, ioba += (1ULL << stt->page_shift))
kvmppc_rm_tce_put(stt, ioba >> stt->page_shift, tce_value); kvmppc_rm_tce_put(stt, ioba >> stt->page_shift, tce_value);
return H_SUCCESS; invalidate_exit:
list_for_each_entry_lockless(stit, &stt->iommu_tables, next)
iommu_tce_kill_rm(stit->tbl, ioba >> stt->page_shift, npages);
return ret;
} }
/* This can be called in either virtual mode or real mode */ /* This can be called in either virtual mode or real mode */