korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-12-04 09:34:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

iommu/arm-smmu: Abstract context bank accesses

Browse Source 
Context bank accesses are fiddly enough to deserve a number of extra
helpers to keep the callsites looking sane, even though there are only
one or two of each.

Signed-off-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Robin Murphy 2019-08-15 19:37:30 +01:00 committed by Will Deacon
parent aadbf2143a
commit 19713fd40d
1 changed files with 73 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
drivers/iommu/arm-smmu.c
View File

@ -82,9 +82,6 @@
((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \ ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
? 0x400 : 0)) ? 0x400 : 0))
/* Translation context bank */
#define ARM_SMMU_CB(smmu, n) ((smmu)->base + (((smmu)->numpage + (n)) << (smmu)->pgshift))
#define MSI_IOVA_BASE 0x8000000 #define MSI_IOVA_BASE 0x8000000
#define MSI_IOVA_LENGTH 0x100000 #define MSI_IOVA_LENGTH 0x100000
@ -265,13 +262,34 @@ static void arm_smmu_writel(struct arm_smmu_device *smmu, int page, int offset,
writel_relaxed(val, arm_smmu_page(smmu, page) + offset); writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
} }
static u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
{
return readq_relaxed(arm_smmu_page(smmu, page) + offset);
}
static void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, int offset,
u64 val)
{
writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
}
#define ARM_SMMU_GR1 1 #define ARM_SMMU_GR1 1
#define ARM_SMMU_CB(s, n) ((s)->numpage + (n))
#define arm_smmu_gr1_read(s, o) \ #define arm_smmu_gr1_read(s, o) \
arm_smmu_readl((s), ARM_SMMU_GR1, (o)) arm_smmu_readl((s), ARM_SMMU_GR1, (o))
#define arm_smmu_gr1_write(s, o, v) \ #define arm_smmu_gr1_write(s, o, v) \
arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v)) arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
#define arm_smmu_cb_read(s, n, o) \
arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
#define arm_smmu_cb_write(s, n, o, v) \
arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
#define arm_smmu_cb_readq(s, n, o) \
arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
#define arm_smmu_cb_writeq(s, n, o, v) \
arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
struct arm_smmu_option_prop { struct arm_smmu_option_prop {
u32 opt; u32 opt;
const char *prop; const char *prop;
@ -427,15 +445,17 @@ static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
} }
/* Wait for any pending TLB invalidations to complete */ /* Wait for any pending TLB invalidations to complete */
static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
void __iomem *sync, void __iomem *status) int sync, int status)
{ {
unsigned int spin_cnt, delay; unsigned int spin_cnt, delay;
u32 reg;
writel_relaxed(QCOM_DUMMY_VAL, sync); arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE)) reg = arm_smmu_readl(smmu, page, status);
if (!(reg & sTLBGSTATUS_GSACTIVE))
return; return;
cpu_relax(); cpu_relax();
} }
@ -447,12 +467,11 @@ static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
{ {
void __iomem *base = ARM_SMMU_GR0(smmu);
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&smmu->global_sync_lock, flags); spin_lock_irqsave(&smmu->global_sync_lock, flags);
__arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC, __arm_smmu_tlb_sync(smmu, 0, ARM_SMMU_GR0_sTLBGSYNC,
base + ARM_SMMU_GR0_sTLBGSTATUS); ARM_SMMU_GR0_sTLBGSTATUS);
spin_unlock_irqrestore(&smmu->global_sync_lock, flags); spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
} }
@ -460,12 +479,11 @@ static void arm_smmu_tlb_sync_context(void *cookie)
{ {
struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_domain *smmu_domain = cookie;
struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_device *smmu = smmu_domain->smmu;
void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&smmu_domain->cb_lock, flags); spin_lock_irqsave(&smmu_domain->cb_lock, flags);
__arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC, __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
base + ARM_SMMU_CB_TLBSTATUS); ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
} }
@ -479,14 +497,13 @@ static void arm_smmu_tlb_sync_vmid(void *cookie)
static void arm_smmu_tlb_inv_context_s1(void *cookie) static void arm_smmu_tlb_inv_context_s1(void *cookie)
{ {
struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_domain *smmu_domain = cookie;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
/* /*
* NOTE: this is not a relaxed write; it needs to guarantee that PTEs * The TLBI write may be relaxed, so ensure that PTEs cleared by the
* cleared by the current CPU are visible to the SMMU before the TLBI. * current CPU are visible beforehand.
*/ */
writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID); wmb();
arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
arm_smmu_tlb_sync_context(cookie); arm_smmu_tlb_sync_context(cookie);
} }
@ -507,25 +524,25 @@ static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_domain *smmu_domain = cookie;
struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
void __iomem *reg = ARM_SMMU_CB(smmu, cfg->cbndx); int reg, idx = cfg->cbndx;
if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
wmb(); wmb();
reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
iova = (iova >> 12) << 12; iova = (iova >> 12) << 12;
iova |= cfg->asid; iova |= cfg->asid;
do { do {
writel_relaxed(iova, reg); arm_smmu_cb_write(smmu, idx, reg, iova);
iova += granule; iova += granule;
} while (size -= granule); } while (size -= granule);
} else { } else {
iova >>= 12; iova >>= 12;
iova |= (u64)cfg->asid << 48; iova |= (u64)cfg->asid << 48;
do { do {
writeq_relaxed(iova, reg); arm_smmu_cb_writeq(smmu, idx, reg, iova);
iova += granule >> 12; iova += granule >> 12;
} while (size -= granule); } while (size -= granule);
} }
@ -536,18 +553,18 @@ static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
{ {
struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_domain *smmu_domain = cookie;
struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_device *smmu = smmu_domain->smmu;
void __iomem *reg = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx); int reg, idx = smmu_domain->cfg.cbndx;
if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
wmb(); wmb();
reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2; reg = leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2;
iova >>= 12; iova >>= 12;
do { do {
if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64) if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
writeq_relaxed(iova, reg); arm_smmu_cb_writeq(smmu, idx, reg, iova);
else else
writel_relaxed(iova, reg); arm_smmu_cb_write(smmu, idx, reg, iova);
iova += granule >> 12; iova += granule >> 12;
} while (size -= granule); } while (size -= granule);
} }
@ -594,25 +611,22 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
unsigned long iova; unsigned long iova;
struct iommu_domain *domain = dev; struct iommu_domain *domain = dev;
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_device *smmu = smmu_domain->smmu;
void __iomem *cb_base; int idx = smmu_domain->cfg.cbndx;
cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
if (!(fsr & FSR_FAULT)) if (!(fsr & FSR_FAULT))
return IRQ_NONE; return IRQ_NONE;
fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
dev_err_ratelimited(smmu->dev, dev_err_ratelimited(smmu->dev,
"Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
fsr, iova, fsynr, cbfrsynra, cfg->cbndx); fsr, iova, fsynr, cbfrsynra, idx);
writel(fsr, cb_base + ARM_SMMU_CB_FSR); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
return IRQ_HANDLED; return IRQ_HANDLED;
} }
@ -697,13 +711,10 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
bool stage1; bool stage1;
struct arm_smmu_cb *cb = &smmu->cbs[idx]; struct arm_smmu_cb *cb = &smmu->cbs[idx];
struct arm_smmu_cfg *cfg = cb->cfg; struct arm_smmu_cfg *cfg = cb->cfg;
void __iomem *cb_base;
cb_base = ARM_SMMU_CB(smmu, idx);
/* Unassigned context banks only need disabling */ /* Unassigned context banks only need disabling */
if (!cfg) { if (!cfg) {
writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
return; return;
} }
@ -746,24 +757,25 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
* access behaviour of some fields (in particular, ASID[15:8]). * access behaviour of some fields (in particular, ASID[15:8]).
*/ */
if (stage1 && smmu->version > ARM_SMMU_V1) if (stage1 && smmu->version > ARM_SMMU_V1)
writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TCR2); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TCR); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
/* TTBRs */ /* TTBRs */
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
} else { } else {
writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0); arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
if (stage1) if (stage1)
writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1); arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
cb->ttbr[1]);
} }
/* MAIRs (stage-1 only) */ /* MAIRs (stage-1 only) */
if (stage1) { if (stage1) {
writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
} }
/* SCTLR */ /* SCTLR */
@ -773,7 +785,7 @@ static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
reg |= SCTLR_E; reg |= SCTLR_E;
writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
} }
static int arm_smmu_init_domain_context(struct iommu_domain *domain, static int arm_smmu_init_domain_context(struct iommu_domain *domain,
@ -1370,27 +1382,25 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
struct device *dev = smmu->dev; struct device *dev = smmu->dev;
void __iomem *cb_base; void __iomem *reg;
u32 tmp; u32 tmp;
u64 phys; u64 phys;
unsigned long va, flags; unsigned long va, flags;
int ret; int ret, idx = cfg->cbndx;
ret = arm_smmu_rpm_get(smmu); ret = arm_smmu_rpm_get(smmu);
if (ret < 0) if (ret < 0)
return 0; return 0;
cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
spin_lock_irqsave(&smmu_domain->cb_lock, flags); spin_lock_irqsave(&smmu_domain->cb_lock, flags);
va = iova & ~0xfffUL; va = iova & ~0xfffUL;
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
writeq_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR); arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
else else
writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR); arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
!(tmp & ATSR_ACTIVE), 5, 50)) { if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) {
spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
dev_err(dev, dev_err(dev,
"iova to phys timed out on %pad. Falling back to software table walk.\n", "iova to phys timed out on %pad. Falling back to software table walk.\n",
@ -1398,7 +1408,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
return ops->iova_to_phys(ops, iova); return ops->iova_to_phys(ops, iova);
} }
phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR); phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
if (phys & CB_PAR_F) { if (phys & CB_PAR_F) {
dev_err(dev, "translation fault!\n"); dev_err(dev, "translation fault!\n");
@ -1762,18 +1772,16 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
/* Make sure all context banks are disabled and clear CB_FSR */ /* Make sure all context banks are disabled and clear CB_FSR */
for (i = 0; i < smmu->num_context_banks; ++i) { for (i = 0; i < smmu->num_context_banks; ++i) {
void __iomem *cb_base = ARM_SMMU_CB(smmu, i);
arm_smmu_write_context_bank(smmu, i); arm_smmu_write_context_bank(smmu, i);
writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR); arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, FSR_FAULT);
/* /*
* Disable MMU-500's not-particularly-beneficial next-page * Disable MMU-500's not-particularly-beneficial next-page
* prefetcher for the sake of errata #841119 and #826419. * prefetcher for the sake of errata #841119 and #826419.
*/ */
if (smmu->model == ARM_MMU500) { if (smmu->model == ARM_MMU500) {
reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR); reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
reg &= ~ARM_MMU500_ACTLR_CPRE; reg &= ~ARM_MMU500_ACTLR_CPRE;
writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR); arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg);
} }
} }