From 27790398c2aed917828dc3c6f81240d57f1584c9 Mon Sep 17 00:00:00 2001 From: Scott Wood Date: Sun, 21 Jan 2018 03:28:54 -0600 Subject: [PATCH 01/53] iommu/amd: Use raw locks on atomic context paths Several functions in this driver are called from atomic context, and thus raw locks must be used in order to be safe on PREEMPT_RT. This includes paths that must wait for command completion, which is a potential PREEMPT_RT latency concern but not easily avoidable. Signed-off-by: Scott Wood Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 30 +++++++++++++++--------------- drivers/iommu/amd_iommu_init.c | 2 +- drivers/iommu/amd_iommu_types.h | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 74788fdeb773..d227f761cc21 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1056,9 +1056,9 @@ static int iommu_queue_command_sync(struct amd_iommu *iommu, unsigned long flags; int ret; - spin_lock_irqsave(&iommu->lock, flags); + raw_spin_lock_irqsave(&iommu->lock, flags); ret = __iommu_queue_command_sync(iommu, cmd, sync); - spin_unlock_irqrestore(&iommu->lock, flags); + raw_spin_unlock_irqrestore(&iommu->lock, flags); return ret; } @@ -1084,7 +1084,7 @@ static int iommu_completion_wait(struct amd_iommu *iommu) build_completion_wait(&cmd, (u64)&iommu->cmd_sem); - spin_lock_irqsave(&iommu->lock, flags); + raw_spin_lock_irqsave(&iommu->lock, flags); iommu->cmd_sem = 0; @@ -1095,7 +1095,7 @@ static int iommu_completion_wait(struct amd_iommu *iommu) ret = wait_on_sem(&iommu->cmd_sem); out_unlock: - spin_unlock_irqrestore(&iommu->lock, flags); + raw_spin_unlock_irqrestore(&iommu->lock, flags); return ret; } @@ -3627,7 +3627,7 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic) goto out_unlock; /* Initialize table spin-lock */ - spin_lock_init(&table->lock); + raw_spin_lock_init(&table->lock); if (ioapic) /* Keep the first 32 indexes free for IOAPIC interrupts */ @@ -3689,7 +3689,7 @@ static int alloc_irq_index(u16 devid, int count, bool align) if (align) alignment = roundup_pow_of_two(count); - spin_lock_irqsave(&table->lock, flags); + raw_spin_lock_irqsave(&table->lock, flags); /* Scan table for free entries */ for (index = ALIGN(table->min_index, alignment), c = 0; @@ -3716,7 +3716,7 @@ static int alloc_irq_index(u16 devid, int count, bool align) index = -ENOSPC; out: - spin_unlock_irqrestore(&table->lock, flags); + raw_spin_unlock_irqrestore(&table->lock, flags); return index; } @@ -3737,7 +3737,7 @@ static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte, if (!table) return -ENOMEM; - spin_lock_irqsave(&table->lock, flags); + raw_spin_lock_irqsave(&table->lock, flags); entry = (struct irte_ga *)table->table; entry = &entry[index]; @@ -3748,7 +3748,7 @@ static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte, if (data) data->ref = entry; - spin_unlock_irqrestore(&table->lock, flags); + raw_spin_unlock_irqrestore(&table->lock, flags); iommu_flush_irt(iommu, devid); iommu_completion_wait(iommu); @@ -3770,9 +3770,9 @@ static int modify_irte(u16 devid, int index, union irte *irte) if (!table) return -ENOMEM; - spin_lock_irqsave(&table->lock, flags); + raw_spin_lock_irqsave(&table->lock, flags); table->table[index] = irte->val; - spin_unlock_irqrestore(&table->lock, flags); + raw_spin_unlock_irqrestore(&table->lock, flags); iommu_flush_irt(iommu, devid); iommu_completion_wait(iommu); @@ -3794,9 +3794,9 @@ static void free_irte(u16 devid, int index) if (!table) return; - spin_lock_irqsave(&table->lock, flags); + raw_spin_lock_irqsave(&table->lock, flags); iommu->irte_ops->clear_allocated(table, index); - spin_unlock_irqrestore(&table->lock, flags); + raw_spin_unlock_irqrestore(&table->lock, flags); iommu_flush_irt(iommu, devid); iommu_completion_wait(iommu); @@ -4397,7 +4397,7 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) if (!irt) return -ENODEV; - spin_lock_irqsave(&irt->lock, flags); + raw_spin_lock_irqsave(&irt->lock, flags); if (ref->lo.fields_vapic.guest_mode) { if (cpu >= 0) @@ -4406,7 +4406,7 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) barrier(); } - spin_unlock_irqrestore(&irt->lock, flags); + raw_spin_unlock_irqrestore(&irt->lock, flags); iommu_flush_irt(iommu, devid); iommu_completion_wait(iommu); diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 4e4a615bf13f..904c575d1677 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -1474,7 +1474,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) { int ret; - spin_lock_init(&iommu->lock); + raw_spin_lock_init(&iommu->lock); /* Add IOMMU to internal data structures */ list_add_tail(&iommu->list, &amd_iommu_list); diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index 6a877ebd058b..da886b0095aa 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -408,7 +408,7 @@ extern bool amd_iommu_iotlb_sup; #define IRQ_TABLE_ALIGNMENT 128 struct irq_remap_table { - spinlock_t lock; + raw_spinlock_t lock; unsigned min_index; u32 *table; }; @@ -490,7 +490,7 @@ struct amd_iommu { int index; /* locks the accesses to the hardware */ - spinlock_t lock; + raw_spinlock_t lock; /* Pointer to PCI device of this IOMMU */ struct pci_dev *dev; From 01ee04badefd296eb7a4430497373be9b7b16783 Mon Sep 17 00:00:00 2001 From: Scott Wood Date: Sun, 28 Jan 2018 14:22:19 -0600 Subject: [PATCH 02/53] iommu/amd: Don't use dev_data in irte_ga_set_affinity() search_dev_data() acquires a non-raw lock, which can't be done from atomic context on PREEMPT_RT. There is no need to look at dev_data because guest_mode should never be set if use_vapic is not set. Signed-off-by: Scott Wood Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index d227f761cc21..d666246ac30b 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3877,10 +3877,8 @@ static void irte_ga_set_affinity(void *entry, u16 devid, u16 index, u8 vector, u32 dest_apicid) { struct irte_ga *irte = (struct irte_ga *) entry; - struct iommu_dev_data *dev_data = search_dev_data(devid); - if (!dev_data || !dev_data->use_vapic || - !irte->lo.fields_remap.guest_mode) { + if (!irte->lo.fields_remap.guest_mode) { irte->hi.fields.vector = vector; irte->lo.fields_remap.destination = dest_apicid; modify_irte_ga(devid, index, irte, NULL); From 6d7cf02a8660cf5570b20017df4b14a2ddbd1694 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 24 Jan 2018 14:22:09 +0000 Subject: [PATCH 03/53] iommu/exynos: Use generic group callback Since iommu_group_get_for_dev() already tries iommu_group_get() and will not call ops->device_group if the group is already non-NULL, the check in get_device_iommu_group() is always redundant and it reduces to a duplicate of the generic version; let's just use that one instead. Signed-off-by: Robin Murphy Tested-by: Marek Szyprowski Signed-off-by: Joerg Roedel --- drivers/iommu/exynos-iommu.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 2138102ef611..210715f08cf6 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1238,17 +1238,6 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, return phys; } -static struct iommu_group *get_device_iommu_group(struct device *dev) -{ - struct iommu_group *group; - - group = iommu_group_get(dev); - if (!group) - group = iommu_group_alloc(); - - return group; -} - static int exynos_iommu_add_device(struct device *dev) { struct exynos_iommu_owner *owner = dev->archdata.iommu; @@ -1344,7 +1333,7 @@ static const struct iommu_ops exynos_iommu_ops = { .unmap = exynos_iommu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = exynos_iommu_iova_to_phys, - .device_group = get_device_iommu_group, + .device_group = generic_device_group, .add_device = exynos_iommu_add_device, .remove_device = exynos_iommu_remove_device, .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, From f3e827d73ed454053d2ce6a4bf021b5adde3ac54 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Fri, 26 Jan 2018 15:11:28 +0800 Subject: [PATCH 04/53] iommu/mediatek: Move attach_device after iommu-group is ready for M4Uv1 In the commit 05f80300dc8b ("iommu: Finish making iommu_group support mandatory"), the iommu framework has supposed all the iommu drivers have their owner iommu-group, it get rid of the FIXME workarounds while the group is NULL. But the flow of Mediatek M4U gen1 looks a bit trick that it will hang at this case: ========================================== Unable to handle kernel NULL pointer dereference at virtual address 00000030 pgd = c0004000 [00000030] *pgd=00000000 PC is at mutex_lock+0x28/0x54 LR is at iommu_attach_device+0xa4/0xd4 pc : [] lr : [] psr: 60000013 sp : df0edbb8 ip : df0edbc8 fp : df0edbc4 r10: c114da14 r9 : df2a3e40 r8 : 00000003 r7 : df27a210 r6 : df2a90c4 r5 : 00000030 r4 : 00000000 r3 : df0f8000 r2 : fffff000 r1 : df29c610 r0 : 00000030 Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment none xxx (mutex_lock) from [] (iommu_attach_device+0xa4/0xd4) (iommu_attach_device) from [] (__arm_iommu_attach_device+0x28/0x90) (__arm_iommu_attach_device) from [] (arm_iommu_attach_device+0x1c/0x30) (arm_iommu_attach_device) from [] (mtk_iommu_add_device+0xfc/0x214) (mtk_iommu_add_device) from [] (add_iommu_group+0x3c/0x68) (add_iommu_group) from [] (bus_for_each_dev+0x78/0xac) (bus_for_each_dev) from [] (bus_set_iommu+0xb0/0xec) (bus_set_iommu) from [] (mtk_iommu_probe+0x328/0x368) (mtk_iommu_probe) from [] (platform_drv_probe+0x5c/0xc0) (platform_drv_probe) from [] (driver_probe_device+0x2f4/0x4d8) (driver_probe_device) from [] (__driver_attach+0x10c/0x128) (__driver_attach) from [] (bus_for_each_dev+0x78/0xac) (bus_for_each_dev) from [] (driver_attach+0x2c/0x30) (driver_attach) from [] (bus_add_driver+0x1e0/0x278) (bus_add_driver) from [] (driver_register+0x88/0x108) (driver_register) from [] (__platform_driver_register+0x50/0x58) (__platform_driver_register) from [] (m4u_init+0x24/0x28) (m4u_init) from [] (do_one_initcall+0xf0/0x17c) ========================= The root cause is that the device's iommu-group is NULL while arm_iommu_attach_device is called. This patch prepare a new iommu-group for the iommu consumer devices to fix this issue. CC: Robin Murphy CC: Honghui Zhang Fixes: 05f80300dc8b ("iommu: Finish making iommu_group support mandatory") Reported-by: Ryder Lee Signed-off-by: Yong Wu Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu_v1.c | 54 ++++++++++++++---------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 542930cd183d..1b1b77594897 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -418,20 +418,12 @@ static int mtk_iommu_create_mapping(struct device *dev, m4udev->archdata.iommu = mtk_mapping; } - ret = arm_iommu_attach_device(dev, mtk_mapping); - if (ret) - goto err_release_mapping; - return 0; - -err_release_mapping: - arm_iommu_release_mapping(mtk_mapping); - m4udev->archdata.iommu = NULL; - return ret; } static int mtk_iommu_add_device(struct device *dev) { + struct dma_iommu_mapping *mtk_mapping; struct of_phandle_args iommu_spec; struct of_phandle_iterator it; struct mtk_iommu_data *data; @@ -452,15 +444,30 @@ static int mtk_iommu_add_device(struct device *dev) if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops) return -ENODEV; /* Not a iommu client device */ - data = dev->iommu_fwspec->iommu_priv; - iommu_device_link(&data->iommu, dev); - - group = iommu_group_get_for_dev(dev); + /* + * This is a short-term bodge because the ARM DMA code doesn't + * understand multi-device groups, but we have to call into it + * successfully (and not just rely on a normal IOMMU API attach + * here) in order to set the correct DMA API ops on @dev. + */ + group = iommu_group_alloc(); if (IS_ERR(group)) return PTR_ERR(group); + err = iommu_group_add_device(group, dev); iommu_group_put(group); - return 0; + if (err) + return err; + + data = dev->iommu_fwspec->iommu_priv; + mtk_mapping = data->dev->archdata.iommu; + err = arm_iommu_attach_device(dev, mtk_mapping); + if (err) { + iommu_group_remove_device(dev); + return err; + } + + return iommu_device_link(&data->iommu, dev);; } static void mtk_iommu_remove_device(struct device *dev) @@ -477,24 +484,6 @@ static void mtk_iommu_remove_device(struct device *dev) iommu_fwspec_free(dev); } -static struct iommu_group *mtk_iommu_device_group(struct device *dev) -{ - struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; - - if (!data) - return ERR_PTR(-ENODEV); - - /* All the client devices are in the same m4u iommu-group */ - if (!data->m4u_group) { - data->m4u_group = iommu_group_alloc(); - if (IS_ERR(data->m4u_group)) - dev_err(dev, "Failed to allocate M4U IOMMU group\n"); - } else { - iommu_group_ref_get(data->m4u_group); - } - return data->m4u_group; -} - static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) { u32 regval; @@ -547,7 +536,6 @@ static struct iommu_ops mtk_iommu_ops = { .iova_to_phys = mtk_iommu_iova_to_phys, .add_device = mtk_iommu_add_device, .remove_device = mtk_iommu_remove_device, - .device_group = mtk_iommu_device_group, .pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT, }; From d15a339eaca6fd6eb00ebc36edf3c289b65000fe Mon Sep 17 00:00:00 2001 From: Dmitry Safonov Date: Mon, 12 Feb 2018 16:48:20 +0000 Subject: [PATCH 05/53] iommu/vt-d: Add __init for dmar_register_bus_notifier() It's called only from intel_iommu_init(), which is init function. Signed-off-by: Dmitry Safonov Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 9a7ffd13c7f0..accf58388bdb 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -806,7 +806,7 @@ int __init dmar_dev_scope_init(void) return dmar_dev_scope_status; } -void dmar_register_bus_notifier(void) +void __init dmar_register_bus_notifier(void) { bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb); } From b1d03c1d12abbfa7de127772f281b309cf1650c3 Mon Sep 17 00:00:00 2001 From: Dmitry Safonov Date: Mon, 12 Feb 2018 16:48:21 +0000 Subject: [PATCH 06/53] iommu/vt-d: Clean/document fault status flags So one could decode them without opening the specification. Signed-off-by: Dmitry Safonov Signed-off-by: Joerg Roedel --- include/linux/intel-iommu.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 8dad3dd26eae..ef169d67df92 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -209,12 +209,12 @@ #define DMA_FECTL_IM (((u32)1) << 31) /* FSTS_REG */ -#define DMA_FSTS_PPF ((u32)2) -#define DMA_FSTS_PFO ((u32)1) -#define DMA_FSTS_IQE (1 << 4) -#define DMA_FSTS_ICE (1 << 5) -#define DMA_FSTS_ITE (1 << 6) -#define DMA_FSTS_PRO (1 << 7) +#define DMA_FSTS_PFO (1 << 0) /* Primary Fault Overflow */ +#define DMA_FSTS_PPF (1 << 1) /* Primary Pending Fault */ +#define DMA_FSTS_IQE (1 << 4) /* Invalidation Queue Error */ +#define DMA_FSTS_ICE (1 << 5) /* Invalidation Completion Error */ +#define DMA_FSTS_ITE (1 << 6) /* Invalidation Time-out Error */ +#define DMA_FSTS_PRO (1 << 7) /* Page Request Overflow */ #define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff) /* FRCD_REG, 32 bits access */ From c5611a8751e67595e4e7d3feaff3c900b92094b9 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Mon, 5 Feb 2018 05:45:53 -0500 Subject: [PATCH 07/53] iommu: Do not return error code for APIs with size_t return type Currently, iommu_unmap, iommu_unmap_fast and iommu_map_sg return size_t. However, some of the return values are error codes (< 0), which can be misinterpreted as large size. Therefore, returning size 0 instead to signify failure to map/unmap. Cc: Joerg Roedel Cc: Alex Williamson Signed-off-by: Suravee Suthikulpanit Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 2 +- drivers/iommu/iommu.c | 6 +++--- include/linux/iommu.h | 14 +++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 74788fdeb773..ecdeb045deef 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3050,7 +3050,7 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, size_t unmap_size; if (domain->mode == PAGE_MODE_NONE) - return -EINVAL; + return 0; mutex_lock(&domain->api_lock); unmap_size = iommu_unmap_page(domain, iova, page_size); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 69fef991c651..d2aa23202bb9 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1573,10 +1573,10 @@ static size_t __iommu_unmap(struct iommu_domain *domain, if (unlikely(ops->unmap == NULL || domain->pgsize_bitmap == 0UL)) - return -ENODEV; + return 0; if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) - return -EINVAL; + return 0; /* find out the minimum page size supported */ min_pagesz = 1 << __ffs(domain->pgsize_bitmap); @@ -1589,7 +1589,7 @@ static size_t __iommu_unmap(struct iommu_domain *domain, if (!IS_ALIGNED(iova | size, min_pagesz)) { pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", iova, size, min_pagesz); - return -EINVAL; + return 0; } pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 41b8c5757859..19938ee6eb31 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -465,23 +465,23 @@ static inline int iommu_map(struct iommu_domain *domain, unsigned long iova, return -ENODEV; } -static inline int iommu_unmap(struct iommu_domain *domain, unsigned long iova, - size_t size) +static inline size_t iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) { - return -ENODEV; + return 0; } -static inline int iommu_unmap_fast(struct iommu_domain *domain, unsigned long iova, - int gfp_order) +static inline size_t iommu_unmap_fast(struct iommu_domain *domain, + unsigned long iova, int gfp_order) { - return -ENODEV; + return 0; } static inline size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, struct scatterlist *sg, unsigned int nents, int prot) { - return -ENODEV; + return 0; } static inline void iommu_flush_tlb_all(struct iommu_domain *domain) From 193e67c00e33e188d68815fb78806317c77f156c Mon Sep 17 00:00:00 2001 From: Vivek Gautam Date: Mon, 5 Feb 2018 23:29:19 +0530 Subject: [PATCH 08/53] iommu/io-pgtable: Use size_t return type for all foo_unmap Unmap returns a size_t all throughout the IOMMU framework. Make io-pgtable match this convention. Moreover, there isn't a need to have a signed int return type as we return 0 in case of failures. Signed-off-by: Vivek Gautam Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm-v7s.c | 21 +++++++++++---------- drivers/iommu/io-pgtable-arm.c | 24 ++++++++++++------------ drivers/iommu/io-pgtable.h | 4 ++-- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 2ca08dc9331c..10e4a3d11c02 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -357,8 +357,8 @@ static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl) return false; } -static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *, unsigned long, - size_t, int, arm_v7s_iopte *); +static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *, unsigned long, + size_t, int, arm_v7s_iopte *); static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data, unsigned long iova, phys_addr_t paddr, int prot, @@ -541,9 +541,10 @@ static arm_v7s_iopte arm_v7s_split_cont(struct arm_v7s_io_pgtable *data, return pte; } -static int arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, - unsigned long iova, size_t size, - arm_v7s_iopte blk_pte, arm_v7s_iopte *ptep) +static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, + unsigned long iova, size_t size, + arm_v7s_iopte blk_pte, + arm_v7s_iopte *ptep) { struct io_pgtable_cfg *cfg = &data->iop.cfg; arm_v7s_iopte pte, *tablep; @@ -584,9 +585,9 @@ static int arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, return size; } -static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, - unsigned long iova, size_t size, int lvl, - arm_v7s_iopte *ptep) +static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, + unsigned long iova, size_t size, int lvl, + arm_v7s_iopte *ptep) { arm_v7s_iopte pte[ARM_V7S_CONT_PAGES]; struct io_pgtable *iop = &data->iop; @@ -656,8 +657,8 @@ static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, return __arm_v7s_unmap(data, iova, size, lvl + 1, ptep); } -static int arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova, - size_t size) +static size_t arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size) { struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 51e5c43caed1..8faaaa04d7a7 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -268,9 +268,9 @@ static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, __arm_lpae_sync_pte(ptep, cfg); } -static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, - unsigned long iova, size_t size, int lvl, - arm_lpae_iopte *ptep); +static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, + unsigned long iova, size_t size, int lvl, + arm_lpae_iopte *ptep); static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, phys_addr_t paddr, arm_lpae_iopte prot, @@ -506,10 +506,10 @@ static void arm_lpae_free_pgtable(struct io_pgtable *iop) kfree(data); } -static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, - unsigned long iova, size_t size, - arm_lpae_iopte blk_pte, int lvl, - arm_lpae_iopte *ptep) +static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, + unsigned long iova, size_t size, + arm_lpae_iopte blk_pte, int lvl, + arm_lpae_iopte *ptep) { struct io_pgtable_cfg *cfg = &data->iop.cfg; arm_lpae_iopte pte, *tablep; @@ -560,9 +560,9 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, return size; } -static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, - unsigned long iova, size_t size, int lvl, - arm_lpae_iopte *ptep) +static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, + unsigned long iova, size_t size, int lvl, + arm_lpae_iopte *ptep) { arm_lpae_iopte pte; struct io_pgtable *iop = &data->iop; @@ -606,8 +606,8 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep); } -static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, - size_t size) +static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size) { struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); arm_lpae_iopte *ptep = data->pgd; diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index cd2e1eafffe6..2df79093cad9 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -119,8 +119,8 @@ struct io_pgtable_cfg { struct io_pgtable_ops { int (*map)(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t size, int prot); - int (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, - size_t size); + size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, + size_t size); phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, unsigned long iova); }; From 8b4282e6b8e239d8ce68ab884c89335cc6fdd7c7 Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Tue, 13 Feb 2018 15:20:50 +0000 Subject: [PATCH 09/53] ACPI/IORT: Add msi address regions reservation helper On some platforms msi parent address regions have to be excluded from normal IOVA allocation in that they are detected and decoded in a HW specific way by system components and so they cannot be considered normal IOVA address space. Add a helper function that retrieves ITS address regions - the msi parent - through IORT device <-> ITS mappings and reserves it so that these regions will not be translated by IOMMU and will be excluded from IOVA allocations. The function checks for the smmu model number and only applies the msi reservation if the platform requires it. Signed-off-by: Shameer Kolothum Reviewed-by: Lorenzo Pieralisi [For the ITS part] Reviewed-by: Marc Zyngier Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/iort.c | 111 ++++++++++++++++++++++++++++++- drivers/irqchip/irq-gic-v3-its.c | 3 +- include/linux/acpi_iort.h | 7 +- 3 files changed, 116 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 95255ecfae7c..e2f7bddf5522 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -39,6 +39,7 @@ struct iort_its_msi_chip { struct list_head list; struct fwnode_handle *fw_node; + phys_addr_t base_addr; u32 translation_id; }; @@ -161,14 +162,16 @@ static LIST_HEAD(iort_msi_chip_list); static DEFINE_SPINLOCK(iort_msi_chip_lock); /** - * iort_register_domain_token() - register domain token and related ITS ID - * to the list from where we can get it back later on. + * iort_register_domain_token() - register domain token along with related + * ITS ID and base address to the list from where we can get it back later on. * @trans_id: ITS ID. + * @base: ITS base address. * @fw_node: Domain token. * * Returns: 0 on success, -ENOMEM if no memory when allocating list element */ -int iort_register_domain_token(int trans_id, struct fwnode_handle *fw_node) +int iort_register_domain_token(int trans_id, phys_addr_t base, + struct fwnode_handle *fw_node) { struct iort_its_msi_chip *its_msi_chip; @@ -178,6 +181,7 @@ int iort_register_domain_token(int trans_id, struct fwnode_handle *fw_node) its_msi_chip->fw_node = fw_node; its_msi_chip->translation_id = trans_id; + its_msi_chip->base_addr = base; spin_lock(&iort_msi_chip_lock); list_add(&its_msi_chip->list, &iort_msi_chip_list); @@ -581,6 +585,24 @@ int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id) return -ENODEV; } +static int __maybe_unused iort_find_its_base(u32 its_id, phys_addr_t *base) +{ + struct iort_its_msi_chip *its_msi_chip; + int ret = -ENODEV; + + spin_lock(&iort_msi_chip_lock); + list_for_each_entry(its_msi_chip, &iort_msi_chip_list, list) { + if (its_msi_chip->translation_id == its_id) { + *base = its_msi_chip->base_addr; + ret = 0; + break; + } + } + spin_unlock(&iort_msi_chip_lock); + + return ret; +} + /** * iort_dev_find_its_id() - Find the ITS identifier for a device * @dev: The device. @@ -766,6 +788,24 @@ static inline bool iort_iommu_driver_enabled(u8 type) } #ifdef CONFIG_IOMMU_API +static struct acpi_iort_node *iort_get_msi_resv_iommu(struct device *dev) +{ + struct acpi_iort_node *iommu; + struct iommu_fwspec *fwspec = dev->iommu_fwspec; + + iommu = iort_get_iort_node(fwspec->iommu_fwnode); + + if (iommu && (iommu->type == ACPI_IORT_NODE_SMMU_V3)) { + struct acpi_iort_smmu_v3 *smmu; + + smmu = (struct acpi_iort_smmu_v3 *)iommu->node_data; + if (smmu->model == ACPI_IORT_SMMU_V3_HISILICON_HI161X) + return iommu; + } + + return NULL; +} + static inline const struct iommu_ops *iort_fwspec_iommu_ops( struct iommu_fwspec *fwspec) { @@ -782,6 +822,69 @@ static inline int iort_add_device_replay(const struct iommu_ops *ops, return err; } + +/** + * iort_iommu_msi_get_resv_regions - Reserved region driver helper + * @dev: Device from iommu_get_resv_regions() + * @head: Reserved region list from iommu_get_resv_regions() + * + * Returns: Number of msi reserved regions on success (0 if platform + * doesn't require the reservation or no associated msi regions), + * appropriate error value otherwise. The ITS interrupt translation + * spaces (ITS_base + SZ_64K, SZ_64K) associated with the device + * are the msi reserved regions. + */ +int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) +{ + struct acpi_iort_its_group *its; + struct acpi_iort_node *iommu_node, *its_node = NULL; + int i, resv = 0; + + iommu_node = iort_get_msi_resv_iommu(dev); + if (!iommu_node) + return 0; + + /* + * Current logic to reserve ITS regions relies on HW topologies + * where a given PCI or named component maps its IDs to only one + * ITS group; if a PCI or named component can map its IDs to + * different ITS groups through IORT mappings this function has + * to be reworked to ensure we reserve regions for all ITS groups + * a given PCI or named component may map IDs to. + */ + + for (i = 0; i < dev->iommu_fwspec->num_ids; i++) { + its_node = iort_node_map_id(iommu_node, + dev->iommu_fwspec->ids[i], + NULL, IORT_MSI_TYPE); + if (its_node) + break; + } + + if (!its_node) + return 0; + + /* Move to ITS specific data */ + its = (struct acpi_iort_its_group *)its_node->node_data; + + for (i = 0; i < its->its_count; i++) { + phys_addr_t base; + + if (!iort_find_its_base(its->identifiers[i], &base)) { + int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; + struct iommu_resv_region *region; + + region = iommu_alloc_resv_region(base + SZ_64K, SZ_64K, + prot, IOMMU_RESV_MSI); + if (region) { + list_add_tail(®ion->list, head); + resv++; + } + } + } + + return (resv == its->its_count) ? resv : -ENODEV; +} #else static inline const struct iommu_ops *iort_fwspec_iommu_ops( struct iommu_fwspec *fwspec) @@ -789,6 +892,8 @@ static inline const struct iommu_ops *iort_fwspec_iommu_ops( static inline int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) { return 0; } +int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) +{ return 0; } #endif static int iort_iommu_xlate(struct device *dev, struct acpi_iort_node *node, diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 06f025fd5726..ab99d1bd7087 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -3450,7 +3450,8 @@ static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header, return -ENOMEM; } - err = iort_register_domain_token(its_entry->translation_id, dom_handle); + err = iort_register_domain_token(its_entry->translation_id, res.start, + dom_handle); if (err) { pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n", &res.start, its_entry->translation_id); diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index 2f7a29242b87..38cd77b39a64 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -26,7 +26,8 @@ #define IORT_IRQ_MASK(irq) (irq & 0xffffffffULL) #define IORT_IRQ_TRIGGER_MASK(irq) ((irq >> 32) & 0xffffffffULL) -int iort_register_domain_token(int trans_id, struct fwnode_handle *fw_node); +int iort_register_domain_token(int trans_id, phys_addr_t base, + struct fwnode_handle *fw_node); void iort_deregister_domain_token(int trans_id); struct fwnode_handle *iort_find_domain_token(int trans_id); #ifdef CONFIG_ACPI_IORT @@ -38,6 +39,7 @@ int iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id); /* IOMMU interface */ void iort_dma_setup(struct device *dev, u64 *dma_addr, u64 *size); const struct iommu_ops *iort_iommu_configure(struct device *dev); +int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head); #else static inline void acpi_iort_init(void) { } static inline u32 iort_msi_map_rid(struct device *dev, u32 req_id) @@ -52,6 +54,9 @@ static inline void iort_dma_setup(struct device *dev, u64 *dma_addr, static inline const struct iommu_ops *iort_iommu_configure( struct device *dev) { return NULL; } +static inline +int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) +{ return 0; } #endif #endif /* __ACPI_IORT_H__ */ From f51dc89265dabad36f1cd832a581106cf4e50158 Mon Sep 17 00:00:00 2001 From: Shameer Kolothum Date: Tue, 13 Feb 2018 15:20:51 +0000 Subject: [PATCH 10/53] iommu/dma: Add HW MSI(GICv3 ITS) address regions reservation Modified iommu_dma_get_resv_regions() to include GICv3 ITS region on ACPI based ARM platfiorms which may require HW MSI reservations. Signed-off-by: Shameer Kolothum Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 25914d36c5ac..f05f3cf90756 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -19,6 +19,7 @@ * along with this program. If not, see . */ +#include #include #include #include @@ -167,13 +168,18 @@ EXPORT_SYMBOL(iommu_put_dma_cookie); * * IOMMU drivers can use this to implement their .get_resv_regions callback * for general non-IOMMU-specific reservations. Currently, this covers host - * bridge windows for PCI devices. + * bridge windows for PCI devices and GICv3 ITS region reservation on ACPI + * based ARM platforms that may require HW MSI reservation. */ void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) { struct pci_host_bridge *bridge; struct resource_entry *window; + if (!is_of_node(dev->iommu_fwspec->iommu_fwnode) && + iort_iommu_msi_get_resv_regions(dev, list) < 0) + return; + if (!dev_is_pci(dev)) return; From df42a04b15f19a842393dc98a84cbc52b1f8ed49 Mon Sep 17 00:00:00 2001 From: Scott Wood Date: Wed, 14 Feb 2018 17:36:28 -0600 Subject: [PATCH 11/53] iommu/amd: Avoid locking get_irq_table() from atomic context get_irq_table() previously acquired amd_iommu_devtable_lock which is not a raw lock, and thus cannot be acquired from atomic context on PREEMPT_RT. Many calls to modify_irte*() come from atomic context due to the IRQ desc->lock, as does amd_iommu_update_ga() due to the preemption disabling in vcpu_load/put(). The only difference between calling get_irq_table() and reading from irq_lookup_table[] directly, other than the lock acquisition and amd_iommu_rlookup_table[] check, is if the table entry is unpopulated, which should never happen when looking up a devid that came from an irq_2_irte struct, as get_irq_table() would have already been called on that devid during irq_remapping_alloc(). The lock acquisition is not needed in these cases because entries in irq_lookup_table[] never change once non-NULL -- nor would the amd_iommu_devtable_lock usage in get_irq_table() provide meaningful protection if they did, since it's released before using the looked up table in the get_irq_table() caller. Rename the old get_irq_table() to alloc_irq_table(), and create a new lockless get_irq_table() to be used in non-allocating contexts that WARNs if it doesn't find what it's looking for. Signed-off-by: Scott Wood Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index d666246ac30b..14efeb306a9f 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3595,7 +3595,22 @@ static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table) amd_iommu_dev_table[devid].data[2] = dte; } -static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic) +static struct irq_remap_table *get_irq_table(u16 devid) +{ + struct irq_remap_table *table; + + if (WARN_ONCE(!amd_iommu_rlookup_table[devid], + "%s: no iommu for devid %x\n", __func__, devid)) + return NULL; + + table = irq_lookup_table[devid]; + if (WARN_ONCE(!table, "%s: no table for devid %x\n", __func__, devid)) + return NULL; + + return table; +} + +static struct irq_remap_table *alloc_irq_table(u16 devid, bool ioapic) { struct irq_remap_table *table = NULL; struct amd_iommu *iommu; @@ -3682,7 +3697,7 @@ static int alloc_irq_index(u16 devid, int count, bool align) if (!iommu) return -ENODEV; - table = get_irq_table(devid, false); + table = alloc_irq_table(devid, false); if (!table) return -ENODEV; @@ -3733,7 +3748,7 @@ static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte, if (iommu == NULL) return -EINVAL; - table = get_irq_table(devid, false); + table = get_irq_table(devid); if (!table) return -ENOMEM; @@ -3766,7 +3781,7 @@ static int modify_irte(u16 devid, int index, union irte *irte) if (iommu == NULL) return -EINVAL; - table = get_irq_table(devid, false); + table = get_irq_table(devid); if (!table) return -ENOMEM; @@ -3790,7 +3805,7 @@ static void free_irte(u16 devid, int index) if (iommu == NULL) return; - table = get_irq_table(devid, false); + table = get_irq_table(devid); if (!table) return; @@ -4108,7 +4123,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, return ret; if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { - if (get_irq_table(devid, true)) + if (alloc_irq_table(devid, true)) index = info->ioapic_pin; else ret = -ENOMEM; @@ -4391,7 +4406,7 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) if (!iommu) return -ENODEV; - irt = get_irq_table(devid, false); + irt = get_irq_table(devid); if (!irt) return -ENODEV; From eb5ecd1a40e2098f805fb63cb07817ac48826e40 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 21 Feb 2018 14:19:45 +0700 Subject: [PATCH 12/53] iommu/amd: Add support for fast IOTLB flushing Since AMD IOMMU driver currently flushes all TLB entries when page size is more than one, use the same interface for both iommu_ops.flush_iotlb_all() and iommu_ops.iotlb_sync(). Cc: Joerg Roedel Signed-off-by: Suravee Suthikulpanit Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 14efeb306a9f..997a947ddc3b 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3056,9 +3056,6 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, unmap_size = iommu_unmap_page(domain, iova, page_size); mutex_unlock(&domain->api_lock); - domain_flush_tlb_pde(domain); - domain_flush_complete(domain); - return unmap_size; } @@ -3176,6 +3173,19 @@ static bool amd_iommu_is_attach_deferred(struct iommu_domain *domain, return dev_data->defer_attach; } +static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain) +{ + struct protection_domain *dom = to_pdomain(domain); + + domain_flush_tlb_pde(dom); + domain_flush_complete(dom); +} + +static void amd_iommu_iotlb_range_add(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ +} + const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, @@ -3194,6 +3204,9 @@ const struct iommu_ops amd_iommu_ops = { .apply_resv_region = amd_iommu_apply_resv_region, .is_attach_deferred = amd_iommu_is_attach_deferred, .pgsize_bitmap = AMD_IOMMU_PGSIZES, + .flush_iotlb_all = amd_iommu_flush_iotlb_all, + .iotlb_range_add = amd_iommu_iotlb_range_add, + .iotlb_sync = amd_iommu_flush_iotlb_all, }; /***************************************************************************** From 1a4e90f25b2ceca3152efa6f7a9e0565c52e34c1 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 20 Feb 2018 20:25:04 +0000 Subject: [PATCH 13/53] iommu/rockchip: Perform a reset on shutdown Trying to do a kexec whilst the iommus are still on is proving to be a challenging exercise. It is terribly unsafe, as we're reusing the memory allocated for the page tables, leading to a likely crash. Let's implement a shutdown method that will at least try to stop DMA from going crazy behind our back. Note that we need to be extra cautious when doing so, as the IOMMU may not be clocked if controlled by a another master, as typical on Rockchip system. Suggested-by: Robin Murphy Signed-off-by: Marc Zyngier Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 9d991c2d8767..6a3719e118da 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1209,6 +1209,23 @@ static int rk_iommu_remove(struct platform_device *pdev) return 0; } +static void rk_iommu_shutdown(struct platform_device *pdev) +{ + struct rk_iommu *iommu = platform_get_drvdata(pdev); + + /* + * Be careful not to try to shutdown an otherwise unused + * IOMMU, as it is likely not to be clocked, and accessing it + * would just block. An IOMMU without a domain is likely to be + * unused, so let's use this as a (weak) guard. + */ + if (iommu && iommu->domain) { + rk_iommu_enable_stall(iommu); + rk_iommu_disable_paging(iommu); + rk_iommu_force_reset(iommu); + } +} + static const struct of_device_id rk_iommu_dt_ids[] = { { .compatible = "rockchip,iommu" }, { /* sentinel */ } @@ -1218,6 +1235,7 @@ MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids); static struct platform_driver rk_iommu_driver = { .probe = rk_iommu_probe, .remove = rk_iommu_remove, + .shutdown = rk_iommu_shutdown, .driver = { .name = "rk_iommu", .of_match_table = rk_iommu_dt_ids, From bbe4b3af9d9e3172fb9aa1f8dcdfaedcb381fc64 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Sat, 24 Feb 2018 13:42:27 +0800 Subject: [PATCH 14/53] iommu/vt-d: Fix a potential memory leak A memory block was allocated in intel_svm_bind_mm() but never freed in a failure path. This patch fixes this by free it to avoid memory leakage. Cc: Ashok Raj Cc: Jacob Pan Cc: # v4.4+ Signed-off-by: Lu Baolu Fixes: 2f26e0a9c9860 ('iommu/vt-d: Add basic SVM PASID support') Signed-off-by: Joerg Roedel --- drivers/iommu/intel-svm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index 35a408d0ae4f..3d4b9245542d 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -396,6 +396,7 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ pasid_max - 1, GFP_KERNEL); if (ret < 0) { kfree(svm); + kfree(sdev); goto out; } svm->pasid = ret; From 90ca3859f5ea90050d00e695355934b37357e7bb Mon Sep 17 00:00:00 2001 From: Gary R Hook Date: Thu, 8 Mar 2018 18:34:41 -0600 Subject: [PATCH 15/53] iommu/amd: Use dev_err to send events to the system log Remove printk and use a more preferable error logging function. Signed-off-by: Gary R Hook Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 55 ++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 997a947ddc3b..4cd19f93ca15 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -547,6 +547,7 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, static void iommu_print_event(struct amd_iommu *iommu, void *__evt) { + struct device *dev = iommu->iommu.dev; int type, devid, domid, flags; volatile u32 *event = __evt; int count = 0; @@ -573,53 +574,53 @@ retry: amd_iommu_report_page_fault(devid, domid, address, flags); return; } else { - printk(KERN_ERR "AMD-Vi: Event logged ["); + dev_err(dev, "AMD-Vi: Event logged ["); } switch (type) { case EVENT_TYPE_ILL_DEV: - printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x " - "address=0x%016llx flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), - address, flags); + dev_err(dev, "ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x " + "address=0x%016llx flags=0x%04x]\n", + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + address, flags); dump_dte_entry(devid); break; case EVENT_TYPE_DEV_TAB_ERR: - printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " - "address=0x%016llx flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), - address, flags); + dev_err(dev, "DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " + "address=0x%016llx flags=0x%04x]\n", + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + address, flags); break; case EVENT_TYPE_PAGE_TAB_ERR: - printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x " - "domain=0x%04x address=0x%016llx flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), - domid, address, flags); + dev_err(dev, "PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x " + "domain=0x%04x address=0x%016llx flags=0x%04x]\n", + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + domid, address, flags); break; case EVENT_TYPE_ILL_CMD: - printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); + dev_err(dev, "ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); dump_command(address); break; case EVENT_TYPE_CMD_HARD_ERR: - printk("COMMAND_HARDWARE_ERROR address=0x%016llx " - "flags=0x%04x]\n", address, flags); + dev_err(dev, "COMMAND_HARDWARE_ERROR address=0x%016llx " + "flags=0x%04x]\n", address, flags); break; case EVENT_TYPE_IOTLB_INV_TO: - printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x " - "address=0x%016llx]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), - address); + dev_err(dev, "IOTLB_INV_TIMEOUT device=%02x:%02x.%x " + "address=0x%016llx]\n", + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + address); break; case EVENT_TYPE_INV_DEV_REQ: - printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x " - "address=0x%016llx flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), - address, flags); + dev_err(dev, "INVALID_DEVICE_REQUEST device=%02x:%02x.%x " + "address=0x%016llx flags=0x%04x]\n", + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + address, flags); break; default: - printk(KERN_ERR "UNKNOWN type=0x%02x event[0]=0x%08x " - "event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n", - type, event[0], event[1], event[2], event[3]); + dev_err(dev, KERN_ERR "UNKNOWN event[0]=0x%08x event[1]=0x%08x " + "event[2]=0x%08x event[3]=0x%08x\n", + event[0], event[1], event[2], event[3]); } memset(__evt, 0, 4 * sizeof(u32)); From b6d57f1da7e81cb056619d41cf015ebe306c296c Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Thu, 1 Mar 2018 19:22:08 +0800 Subject: [PATCH 16/53] iommu/omap: Increase group ref in .device_group() Increase group refcounting in omap_iommu_device_group(). Signed-off-by: Jeffy Chen Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index e135ab830ebf..c33b7b104e72 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1536,7 +1536,7 @@ static struct iommu_group *omap_iommu_device_group(struct device *dev) struct iommu_group *group = ERR_PTR(-EINVAL); if (arch_data->iommu_dev) - group = arch_data->iommu_dev->group; + group = iommu_group_ref_get(arch_data->iommu_dev->group); return group; } From 9fe77e0166ae453d61954e8200b90ac7000242de Mon Sep 17 00:00:00 2001 From: Biju Das Date: Wed, 24 Jan 2018 15:42:00 +0000 Subject: [PATCH 17/53] dt-bindings: iommu: ipmmu-vmsa: Add device tree support for r8a774[35] Document r8a774[35] specific compatible strings. The Renesas RZ/G1[ME] (r8a774[35]) IPMMU are identical to the R-Car Gen2 family. Signed-off-by: Biju Das Reviewed-by: Chris Paterson Reviewed-by: Geert Uytterhoeven Reviewed-by: Simon Horman Signed-off-by: Joerg Roedel --- .../devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt index 1fd5d69647ca..ffadb7c6f1f3 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt @@ -11,6 +11,8 @@ Required Properties: the device is compatible with the R-Car Gen2 VMSA-compatible IPMMU. - "renesas,ipmmu-r8a73a4" for the R8A73A4 (R-Mobile APE6) IPMMU. + - "renesas,ipmmu-r8a7743" for the R8A7743 (RZ/G1M) IPMMU. + - "renesas,ipmmu-r8a7745" for the R8A7745 (RZ/G1E) IPMMU. - "renesas,ipmmu-r8a7790" for the R8A7790 (R-Car H2) IPMMU. - "renesas,ipmmu-r8a7791" for the R8A7791 (R-Car M2-W) IPMMU. - "renesas,ipmmu-r8a7793" for the R8A7793 (R-Car M2-N) IPMMU. @@ -19,7 +21,8 @@ Required Properties: - "renesas,ipmmu-r8a7796" for the R8A7796 (R-Car M3-W) IPMMU. - "renesas,ipmmu-r8a77970" for the R8A77970 (R-Car V3M) IPMMU. - "renesas,ipmmu-r8a77995" for the R8A77995 (R-Car D3) IPMMU. - - "renesas,ipmmu-vmsa" for generic R-Car Gen2 VMSA-compatible IPMMU. + - "renesas,ipmmu-vmsa" for generic R-Car Gen2 or RZ/G1 VMSA-compatible + IPMMU. - reg: Base address and size of the IPMMU registers. - interrupts: Specifiers for the MMU fault interrupts. For instances that From 971401015d14dfbb483aa6385f8bddc1afad0c32 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Fri, 16 Mar 2018 12:31:36 +0800 Subject: [PATCH 18/53] iommu/vt-d: Use real PASID for flush in caching mode If caching mode is supported, the hardware will cache none-present or erroneous translation entries. Hence, software should explicitly invalidate the PASID cache after a PASID table entry becomes present. We should issue such invalidation with the PASID value that we have changed. PASID 0 is not reserved for this case. Cc: Jacob Pan Cc: Kevin Tian Cc: Sankaran Rajesh Suggested-by: Ashok Raj Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-svm.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index 3d4b9245542d..43737d37fdf2 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -423,17 +423,13 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ iommu->pasid_table[svm->pasid].val = pasid_entry_val; wmb(); - /* In caching mode, we still have to flush with PASID 0 when - * a PASID table entry becomes present. Not entirely clear - * *why* that would be the case — surely we could just issue - * a flush with the PASID value that we've changed? The PASID - * is the index into the table, after all. It's not like domain - * IDs in the case of the equivalent context-entry change in - * caching mode. And for that matter it's not entirely clear why - * a VMM would be in the business of caching the PASID table - * anyway. Surely that can be left entirely to the guest? */ + + /* + * Flush PASID cache when a PASID table entry becomes + * present. + */ if (cap_caching_mode(iommu->cap)) - intel_flush_pasid_dev(svm, sdev, 0); + intel_flush_pasid_dev(svm, sdev, svm->pasid); } list_add_rcu(&sdev->list, &svm->devs); From 70ca608b2ec6dafa6bb1c2b0691852fc78f8f717 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Sun, 18 Mar 2018 09:52:54 +0800 Subject: [PATCH 19/53] iommu/mediatek: Fix protect memory setting In MediaTek's IOMMU design, When a iommu translation fault occurs (HW can NOT translate the destination address to a valid physical address), the IOMMU HW output the dirty data into a special memory to avoid corrupting the main memory, this is called "protect memory". the register(0x114) for protect memory is a little different between mt8173 and mt2712. In the mt8173, bit[30:6] in the register represents [31:7] of the physical address. In the 4GB mode, the register bit[31] should be 1. While in the mt2712, the bits don't shift. bit[31:7] in the register represents [31:7] in the physical address, and bit[1:0] in the register represents bit[33:32] of the physical address if it has. Fixes: e6dec9230862 ("iommu/mediatek: Add mt2712 IOMMU support") Reported-by: Honghui Zhang Signed-off-by: Yong Wu Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 15 ++++++++++----- drivers/iommu/mtk_iommu.h | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index f227d73e7bf6..f2832a10fcea 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -60,7 +60,7 @@ (((prot) & 0x3) << F_MMU_TF_PROTECT_SEL_SHIFT(data)) #define REG_MMU_IVRP_PADDR 0x114 -#define F_MMU_IVRP_PA_SET(pa, ext) (((pa) >> 1) | ((!!(ext)) << 31)) + #define REG_MMU_VLD_PA_RNG 0x118 #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA)) @@ -539,8 +539,13 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) F_INT_PRETETCH_TRANSATION_FIFO_FAULT; writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); - writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB), - data->base + REG_MMU_IVRP_PADDR); + if (data->m4u_plat == M4U_MT8173) + regval = (data->protect_base >> 1) | (data->enable_4GB << 31); + else + regval = lower_32_bits(data->protect_base) | + upper_32_bits(data->protect_base); + writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR); + if (data->enable_4GB && data->m4u_plat != M4U_MT8173) { /* * If 4GB mode is enabled, the validate PA range is from @@ -695,6 +700,7 @@ static int __maybe_unused mtk_iommu_suspend(struct device *dev) reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); + reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR); clk_disable_unprepare(data->bclk); return 0; } @@ -717,8 +723,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev) writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); - writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB), - base + REG_MMU_IVRP_PADDR); + writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR); if (data->m4u_dom) writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0], base + REG_MMU_PT_BASE_ADDR); diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index b4451a1c7c2f..778498b8633f 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -32,6 +32,7 @@ struct mtk_iommu_suspend_reg { u32 ctrl_reg; u32 int_control0; u32 int_main_control; + u32 ivrp_paddr; }; enum mtk_iommu_plat { From 4c8996d7d7334c1adfbfdaaddc979094f7811ce6 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 30 Oct 2017 12:14:02 +0000 Subject: [PATCH 20/53] iommu/arm-smmu-v3: Warn about missing IRQs It is annoyingly non-obvious when DMA transactions silently go missing due to undetected SMMU faults. Help skip the first few debugging steps in those situations by making it clear when we have neither wired IRQs nor MSIs with which to raise error conditions. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 3f2f1fc68b52..ebf22e9746f6 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2370,6 +2370,8 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) "arm-smmu-v3-evtq", smmu); if (ret < 0) dev_warn(smmu->dev, "failed to enable evtq irq\n"); + } else { + dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n"); } irq = smmu->gerr_irq; @@ -2378,6 +2380,8 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) 0, "arm-smmu-v3-gerror", smmu); if (ret < 0) dev_warn(smmu->dev, "failed to enable gerror irq\n"); + } else { + dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n"); } if (smmu->features & ARM_SMMU_FEAT_PRI) { @@ -2391,6 +2395,8 @@ static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) if (ret < 0) dev_warn(smmu->dev, "failed to enable priq irq\n"); + } else { + dev_warn(smmu->dev, "no priq irq - PRI will be broken\n"); } } } From 940ded9c21fde7c47c64d5cc74e22300fb89e6b7 Mon Sep 17 00:00:00 2001 From: Nate Watterson Date: Sat, 20 Jan 2018 13:08:04 -0500 Subject: [PATCH 21/53] iommu/arm-smmu-v3: limit reporting of MSI allocation failures Currently, the arm-smmu-v3 driver expects to allocate MSIs for all SMMUs with FEAT_MSI set. This results in unwarranted "failed to allocate MSIs" warnings being printed on systems where FW was either deliberately configured to force the use of SMMU wired interrupts -or- is altogether incapable of describing SMMU MSI topology (ACPI IORT prior to rev.C). Remedy this by checking msi_domain before attempting to allocate SMMU MSIs. Acked-by: Marc Zyngier Signed-off-by: Nate Watterson Signed-off-by: Sinan Kaya Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index ebf22e9746f6..d9b944eabf71 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2328,10 +2328,15 @@ static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) if (!(smmu->features & ARM_SMMU_FEAT_MSI)) return; + if (!dev->msi_domain) { + dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n"); + return; + } + /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */ ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg); if (ret) { - dev_warn(dev, "failed to allocate MSIs\n"); + dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n"); return; } From 1cf9e54e91aac61e1a6c2bbacd0b571c6ca09131 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 26 Mar 2018 13:35:09 +0100 Subject: [PATCH 22/53] iommu/arm-smmu-v3: Clean up address masking Before trying to add the SMMUv3.1 support for 52-bit addresses, make things bearable by cleaning up the various address mask definitions to use GENMASK_ULL() consistently. The fact that doing so reveals (and fixes) a latent off-by-one in Q_BASE_ADDR_MASK only goes to show what a jolly good idea it is... Tested-by: Nate Watterson Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 53 +++++++++++++++---------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index d9b944eabf71..87da25815dac 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -158,8 +159,7 @@ #define ARM_SMMU_STRTAB_BASE 0x80 #define STRTAB_BASE_RA (1UL << 62) -#define STRTAB_BASE_ADDR_SHIFT 6 -#define STRTAB_BASE_ADDR_MASK 0x3ffffffffffUL +#define STRTAB_BASE_ADDR_MASK GENMASK_ULL(47, 6) #define ARM_SMMU_STRTAB_BASE_CFG 0x88 #define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0 @@ -190,8 +190,7 @@ #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc /* Common MSI config fields */ -#define MSI_CFG0_ADDR_SHIFT 2 -#define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL +#define MSI_CFG0_ADDR_MASK GENMASK_ULL(47, 2) #define MSI_CFG2_SH_SHIFT 4 #define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT) #define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT) @@ -207,8 +206,7 @@ Q_IDX(q, p) * (q)->ent_dwords) #define Q_BASE_RWA (1UL << 62) -#define Q_BASE_ADDR_SHIFT 5 -#define Q_BASE_ADDR_MASK 0xfffffffffffUL +#define Q_BASE_ADDR_MASK GENMASK_ULL(47, 5) #define Q_BASE_LOG2SIZE_SHIFT 0 #define Q_BASE_LOG2SIZE_MASK 0x1fUL @@ -225,8 +223,7 @@ #define STRTAB_L1_DESC_DWORDS 1 #define STRTAB_L1_DESC_SPAN_SHIFT 0 #define STRTAB_L1_DESC_SPAN_MASK 0x1fUL -#define STRTAB_L1_DESC_L2PTR_SHIFT 6 -#define STRTAB_L1_DESC_L2PTR_MASK 0x3ffffffffffUL +#define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(47, 6) #define STRTAB_STE_DWORDS 8 #define STRTAB_STE_0_V (1UL << 0) @@ -239,8 +236,7 @@ #define STRTAB_STE_0_S1FMT_SHIFT 4 #define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT) -#define STRTAB_STE_0_S1CTXPTR_SHIFT 6 -#define STRTAB_STE_0_S1CTXPTR_MASK 0x3ffffffffffUL +#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(47, 6) #define STRTAB_STE_0_S1CDMAX_SHIFT 59 #define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL @@ -278,8 +274,7 @@ #define STRTAB_STE_2_S2PTW (1UL << 54) #define STRTAB_STE_2_S2R (1UL << 58) -#define STRTAB_STE_3_S2TTB_SHIFT 4 -#define STRTAB_STE_3_S2TTB_MASK 0xfffffffffffUL +#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(47, 4) /* Context descriptor (stage-1 only) */ #define CTXDESC_CD_DWORDS 8 @@ -325,8 +320,7 @@ #define CTXDESC_CD_0_ASID_SHIFT 48 #define CTXDESC_CD_0_ASID_MASK 0xffffUL -#define CTXDESC_CD_1_TTB0_SHIFT 4 -#define CTXDESC_CD_1_TTB0_MASK 0xfffffffffffUL +#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(47, 4) #define CTXDESC_CD_3_MAIR_SHIFT 0 @@ -351,7 +345,7 @@ #define CMDQ_PREFETCH_0_SID_SHIFT 32 #define CMDQ_PREFETCH_1_SIZE_SHIFT 0 -#define CMDQ_PREFETCH_1_ADDR_MASK ~0xfffUL +#define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12) #define CMDQ_CFGI_0_SID_SHIFT 32 #define CMDQ_CFGI_0_SID_MASK 0xffffffffUL @@ -362,8 +356,8 @@ #define CMDQ_TLBI_0_VMID_SHIFT 32 #define CMDQ_TLBI_0_ASID_SHIFT 48 #define CMDQ_TLBI_1_LEAF (1UL << 0) -#define CMDQ_TLBI_1_VA_MASK ~0xfffUL -#define CMDQ_TLBI_1_IPA_MASK 0xfffffffff000UL +#define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12) +#define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(47, 12) #define CMDQ_PRI_0_SSID_SHIFT 12 #define CMDQ_PRI_0_SSID_MASK 0xfffffUL @@ -386,8 +380,7 @@ #define CMDQ_SYNC_0_MSIATTR_OIWB (0xfUL << CMDQ_SYNC_0_MSIATTR_SHIFT) #define CMDQ_SYNC_0_MSIDATA_SHIFT 32 #define CMDQ_SYNC_0_MSIDATA_MASK 0xffffffffUL -#define CMDQ_SYNC_1_MSIADDR_SHIFT 0 -#define CMDQ_SYNC_1_MSIADDR_MASK 0xffffffffffffcUL +#define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(47, 2) /* Event queue */ #define EVTQ_ENT_DWORDS 4 @@ -413,8 +406,7 @@ #define PRIQ_1_PRG_IDX_SHIFT 0 #define PRIQ_1_PRG_IDX_MASK 0x1ffUL -#define PRIQ_1_ADDR_SHIFT 12 -#define PRIQ_1_ADDR_MASK 0xfffffffffffffUL +#define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12) /* High-level queue structures */ #define ARM_SMMU_POLL_TIMEOUT_US 100 @@ -1093,7 +1085,7 @@ static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu, cfg->cdptr[0] = cpu_to_le64(val); - val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK << CTXDESC_CD_1_TTB0_SHIFT; + val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK; cfg->cdptr[1] = cpu_to_le64(val); cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT); @@ -1107,8 +1099,7 @@ arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK) << STRTAB_L1_DESC_SPAN_SHIFT; - val |= desc->l2ptr_dma & - STRTAB_L1_DESC_L2PTR_MASK << STRTAB_L1_DESC_L2PTR_SHIFT; + val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; *dst = cpu_to_le64(val); } @@ -1214,8 +1205,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); - val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK - << STRTAB_STE_0_S1CTXPTR_SHIFT) | + val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) | STRTAB_STE_0_CFG_S1_TRANS; } @@ -1232,7 +1222,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, STRTAB_STE_2_S2R); dst[3] = cpu_to_le64(ste->s2_cfg->vttbr & - STRTAB_STE_3_S2TTB_MASK << STRTAB_STE_3_S2TTB_SHIFT); + STRTAB_STE_3_S2TTB_MASK); val |= STRTAB_STE_0_CFG_S2_TRANS; } @@ -1337,7 +1327,7 @@ static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) evt[0] & PRIQ_0_PERM_READ ? "R" : "", evt[0] & PRIQ_0_PERM_WRITE ? "W" : "", evt[0] & PRIQ_0_PERM_EXEC ? "X" : "", - evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT); + evt[1] & PRIQ_1_ADDR_MASK); if (last) { struct arm_smmu_cmdq_ent cmd = { @@ -2102,7 +2092,7 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, q->ent_dwords = dwords; q->q_base = Q_BASE_RWA; - q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT; + q->q_base |= q->base_dma & Q_BASE_ADDR_MASK; q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK) << Q_BASE_LOG2SIZE_SHIFT; @@ -2239,8 +2229,7 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) return ret; /* Set the strtab base address */ - reg = smmu->strtab_cfg.strtab_dma & - STRTAB_BASE_ADDR_MASK << STRTAB_BASE_ADDR_SHIFT; + reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK; reg |= STRTAB_BASE_RA; smmu->strtab_cfg.strtab_base = reg; @@ -2303,7 +2292,7 @@ static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index]; doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo; - doorbell &= MSI_CFG0_ADDR_MASK << MSI_CFG0_ADDR_SHIFT; + doorbell &= MSI_CFG0_ADDR_MASK; writeq_relaxed(doorbell, smmu->base + cfg[0]); writel_relaxed(msg->data, smmu->base + cfg[1]); From cbcee19ac4a2c960d73b8951476f0fe5e5628319 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 26 Mar 2018 13:35:10 +0100 Subject: [PATCH 23/53] iommu/arm-smmu-v3: Clean up register definitions The FIELD_{GET,PREP} accessors provided by linux/bitfield.h allow us to define multi-bit register fields solely in terms of their bit positions via GENMASK(), without needing explicit *_SHIFT and *_MASK definitions. As well as the immediate reduction in lines of code, this avoids the awkwardness of values sometimes being pre-shifted and sometimes not, which means we can factor out some common values like memory attributes. Furthermore, it also makes it trivial to verify the definitions against the architecture spec, on which note let's also fix up a few field names to properly match the current release (IHI0070B). Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 174 ++++++++++++++++-------------------- 1 file changed, 77 insertions(+), 97 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 87da25815dac..48531c694c60 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -44,18 +45,15 @@ /* MMIO registers */ #define ARM_SMMU_IDR0 0x0 -#define IDR0_ST_LVL_SHIFT 27 -#define IDR0_ST_LVL_MASK 0x3 -#define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT) -#define IDR0_STALL_MODEL_SHIFT 24 -#define IDR0_STALL_MODEL_MASK 0x3 -#define IDR0_STALL_MODEL_STALL (0 << IDR0_STALL_MODEL_SHIFT) -#define IDR0_STALL_MODEL_FORCE (2 << IDR0_STALL_MODEL_SHIFT) -#define IDR0_TTENDIAN_SHIFT 21 -#define IDR0_TTENDIAN_MASK 0x3 -#define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT) -#define IDR0_TTENDIAN_BE (3 << IDR0_TTENDIAN_SHIFT) -#define IDR0_TTENDIAN_MIXED (0 << IDR0_TTENDIAN_SHIFT) +#define IDR0_ST_LVL GENMASK(28, 27) +#define IDR0_ST_LVL_2LVL 1 +#define IDR0_STALL_MODEL GENMASK(25, 24) +#define IDR0_STALL_MODEL_STALL 0 +#define IDR0_STALL_MODEL_FORCE 2 +#define IDR0_TTENDIAN GENMASK(22, 21) +#define IDR0_TTENDIAN_MIXED 0 +#define IDR0_TTENDIAN_LE 2 +#define IDR0_TTENDIAN_BE 3 #define IDR0_CD2L (1 << 19) #define IDR0_VMID16 (1 << 18) #define IDR0_PRI (1 << 16) @@ -65,10 +63,9 @@ #define IDR0_ATS (1 << 10) #define IDR0_HYP (1 << 9) #define IDR0_COHACC (1 << 4) -#define IDR0_TTF_SHIFT 2 -#define IDR0_TTF_MASK 0x3 -#define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT) -#define IDR0_TTF_AARCH32_64 (3 << IDR0_TTF_SHIFT) +#define IDR0_TTF GENMASK(3, 2) +#define IDR0_TTF_AARCH64 2 +#define IDR0_TTF_AARCH32_64 3 #define IDR0_S1P (1 << 1) #define IDR0_S2P (1 << 0) @@ -76,31 +73,24 @@ #define IDR1_TABLES_PRESET (1 << 30) #define IDR1_QUEUES_PRESET (1 << 29) #define IDR1_REL (1 << 28) -#define IDR1_CMDQ_SHIFT 21 -#define IDR1_CMDQ_MASK 0x1f -#define IDR1_EVTQ_SHIFT 16 -#define IDR1_EVTQ_MASK 0x1f -#define IDR1_PRIQ_SHIFT 11 -#define IDR1_PRIQ_MASK 0x1f -#define IDR1_SSID_SHIFT 6 -#define IDR1_SSID_MASK 0x1f -#define IDR1_SID_SHIFT 0 -#define IDR1_SID_MASK 0x3f +#define IDR1_CMDQS GENMASK(25, 21) +#define IDR1_EVTQS GENMASK(20, 16) +#define IDR1_PRIQS GENMASK(15, 11) +#define IDR1_SSIDSIZE GENMASK(10, 6) +#define IDR1_SIDSIZE GENMASK(5, 0) #define ARM_SMMU_IDR5 0x14 -#define IDR5_STALL_MAX_SHIFT 16 -#define IDR5_STALL_MAX_MASK 0xffff +#define IDR5_STALL_MAX GENMASK(31, 16) #define IDR5_GRAN64K (1 << 6) #define IDR5_GRAN16K (1 << 5) #define IDR5_GRAN4K (1 << 4) -#define IDR5_OAS_SHIFT 0 -#define IDR5_OAS_MASK 0x7 -#define IDR5_OAS_32_BIT (0 << IDR5_OAS_SHIFT) -#define IDR5_OAS_36_BIT (1 << IDR5_OAS_SHIFT) -#define IDR5_OAS_40_BIT (2 << IDR5_OAS_SHIFT) -#define IDR5_OAS_42_BIT (3 << IDR5_OAS_SHIFT) -#define IDR5_OAS_44_BIT (4 << IDR5_OAS_SHIFT) -#define IDR5_OAS_48_BIT (5 << IDR5_OAS_SHIFT) +#define IDR5_OAS GENMASK(2, 0) +#define IDR5_OAS_32_BIT 0 +#define IDR5_OAS_36_BIT 1 +#define IDR5_OAS_40_BIT 2 +#define IDR5_OAS_42_BIT 3 +#define IDR5_OAS_44_BIT 4 +#define IDR5_OAS_48_BIT 5 #define ARM_SMMU_CR0 0x20 #define CR0_CMDQEN (1 << 3) @@ -111,18 +101,16 @@ #define ARM_SMMU_CR0ACK 0x24 #define ARM_SMMU_CR1 0x28 -#define CR1_SH_NSH 0 -#define CR1_SH_OSH 2 -#define CR1_SH_ISH 3 +#define CR1_TABLE_SH GENMASK(11, 10) +#define CR1_TABLE_OC GENMASK(9, 8) +#define CR1_TABLE_IC GENMASK(7, 6) +#define CR1_QUEUE_SH GENMASK(5, 4) +#define CR1_QUEUE_OC GENMASK(3, 2) +#define CR1_QUEUE_IC GENMASK(1, 0) +/* CR1 cacheability fields don't quite follow the usual TCR-style encoding */ #define CR1_CACHE_NC 0 #define CR1_CACHE_WB 1 #define CR1_CACHE_WT 2 -#define CR1_TABLE_SH_SHIFT 10 -#define CR1_TABLE_OC_SHIFT 8 -#define CR1_TABLE_IC_SHIFT 6 -#define CR1_QUEUE_SH_SHIFT 4 -#define CR1_QUEUE_OC_SHIFT 2 -#define CR1_QUEUE_IC_SHIFT 0 #define ARM_SMMU_CR2 0x2c #define CR2_PTM (1 << 2) @@ -130,8 +118,8 @@ #define CR2_E2H (1 << 0) #define ARM_SMMU_GBPA 0x44 -#define GBPA_ABORT (1 << 20) #define GBPA_UPDATE (1 << 31) +#define GBPA_ABORT (1 << 20) #define ARM_SMMU_IRQ_CTRL 0x50 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2) @@ -162,14 +150,11 @@ #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(47, 6) #define ARM_SMMU_STRTAB_BASE_CFG 0x88 -#define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0 -#define STRTAB_BASE_CFG_LOG2SIZE_MASK 0x3f -#define STRTAB_BASE_CFG_SPLIT_SHIFT 6 -#define STRTAB_BASE_CFG_SPLIT_MASK 0x1f -#define STRTAB_BASE_CFG_FMT_SHIFT 16 -#define STRTAB_BASE_CFG_FMT_MASK 0x3 -#define STRTAB_BASE_CFG_FMT_LINEAR (0 << STRTAB_BASE_CFG_FMT_SHIFT) -#define STRTAB_BASE_CFG_FMT_2LVL (1 << STRTAB_BASE_CFG_FMT_SHIFT) +#define STRTAB_BASE_CFG_FMT GENMASK(17, 16) +#define STRTAB_BASE_CFG_FMT_LINEAR 0 +#define STRTAB_BASE_CFG_FMT_2LVL 1 +#define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6) +#define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0) #define ARM_SMMU_CMDQ_BASE 0x90 #define ARM_SMMU_CMDQ_PROD 0x98 @@ -191,12 +176,14 @@ /* Common MSI config fields */ #define MSI_CFG0_ADDR_MASK GENMASK_ULL(47, 2) -#define MSI_CFG2_SH_SHIFT 4 -#define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT) -#define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT) -#define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT) -#define MSI_CFG2_MEMATTR_SHIFT 0 -#define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT) +#define MSI_CFG2_SH GENMASK(5, 4) +#define MSI_CFG2_MEMATTR GENMASK(3, 0) + +/* Common memory attribute values */ +#define ARM_SMMU_SH_NSH 0 +#define ARM_SMMU_SH_OSH 2 +#define ARM_SMMU_SH_ISH 3 +#define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1 #define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1)) #define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift)) @@ -207,8 +194,7 @@ #define Q_BASE_RWA (1UL << 62) #define Q_BASE_ADDR_MASK GENMASK_ULL(47, 5) -#define Q_BASE_LOG2SIZE_SHIFT 0 -#define Q_BASE_LOG2SIZE_MASK 0x1fUL +#define Q_BASE_LOG2SIZE GENMASK(4, 0) /* * Stream table. @@ -333,8 +319,7 @@ #define CMDQ_ENT_DWORDS 2 #define CMDQ_MAX_SZ_SHIFT 8 -#define CMDQ_ERR_SHIFT 24 -#define CMDQ_ERR_MASK 0x7f +#define CMDQ_CONS_ERR GENMASK(30, 24) #define CMDQ_ERR_CERROR_NONE_IDX 0 #define CMDQ_ERR_CERROR_ILL_IDX 1 #define CMDQ_ERR_CERROR_ABT_IDX 2 @@ -910,7 +895,7 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) u64 cmd[CMDQ_ENT_DWORDS]; struct arm_smmu_queue *q = &smmu->cmdq.q; u32 cons = readl_relaxed(q->cons_reg); - u32 idx = cons >> CMDQ_ERR_SHIFT & CMDQ_ERR_MASK; + u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons); struct arm_smmu_cmdq_ent cmd_sync = { .opcode = CMDQ_OP_CMD_SYNC, }; @@ -2093,8 +2078,7 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, q->q_base = Q_BASE_RWA; q->q_base |= q->base_dma & Q_BASE_ADDR_MASK; - q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK) - << Q_BASE_LOG2SIZE_SHIFT; + q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->max_n_shift); q->prod = q->cons = 0; return 0; @@ -2176,11 +2160,9 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) cfg->strtab = strtab; /* Configure strtab_base_cfg for 2 levels */ - reg = STRTAB_BASE_CFG_FMT_2LVL; - reg |= (size & STRTAB_BASE_CFG_LOG2SIZE_MASK) - << STRTAB_BASE_CFG_LOG2SIZE_SHIFT; - reg |= (STRTAB_SPLIT & STRTAB_BASE_CFG_SPLIT_MASK) - << STRTAB_BASE_CFG_SPLIT_SHIFT; + reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL); + reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size); + reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT); cfg->strtab_base_cfg = reg; return arm_smmu_init_l1_strtab(smmu); @@ -2206,9 +2188,8 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) cfg->num_l1_ents = 1 << smmu->sid_bits; /* Configure strtab_base_cfg for a linear table covering all SIDs */ - reg = STRTAB_BASE_CFG_FMT_LINEAR; - reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK) - << STRTAB_BASE_CFG_LOG2SIZE_SHIFT; + reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR); + reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); cfg->strtab_base_cfg = reg; arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents); @@ -2296,7 +2277,7 @@ static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) writeq_relaxed(doorbell, smmu->base + cfg[0]); writel_relaxed(msg->data, smmu->base + cfg[1]); - writel_relaxed(MSI_CFG2_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); + writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); } static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) @@ -2463,12 +2444,12 @@ static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass) return ret; /* CR1 (table and queue memory attributes) */ - reg = (CR1_SH_ISH << CR1_TABLE_SH_SHIFT) | - (CR1_CACHE_WB << CR1_TABLE_OC_SHIFT) | - (CR1_CACHE_WB << CR1_TABLE_IC_SHIFT) | - (CR1_SH_ISH << CR1_QUEUE_SH_SHIFT) | - (CR1_CACHE_WB << CR1_QUEUE_OC_SHIFT) | - (CR1_CACHE_WB << CR1_QUEUE_IC_SHIFT); + reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) | + FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) | + FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) | + FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) | + FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) | + FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB); writel_relaxed(reg, smmu->base + ARM_SMMU_CR1); /* CR2 (random crap) */ @@ -2578,7 +2559,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); /* 2-level structures */ - if ((reg & IDR0_ST_LVL_MASK << IDR0_ST_LVL_SHIFT) == IDR0_ST_LVL_2LVL) + if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL) smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB; if (reg & IDR0_CD2L) @@ -2589,7 +2570,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) * We currently require the same endianness as the CPU, but this * could be changed later by adding a new IO_PGTABLE_QUIRK. */ - switch (reg & IDR0_TTENDIAN_MASK << IDR0_TTENDIAN_SHIFT) { + switch (FIELD_GET(IDR0_TTENDIAN, reg)) { case IDR0_TTENDIAN_MIXED: smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE; break; @@ -2631,7 +2612,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n", coherent ? "true" : "false"); - switch (reg & IDR0_STALL_MODEL_MASK << IDR0_STALL_MODEL_SHIFT) { + switch (FIELD_GET(IDR0_STALL_MODEL, reg)) { case IDR0_STALL_MODEL_FORCE: smmu->features |= ARM_SMMU_FEAT_STALL_FORCE; /* Fallthrough */ @@ -2651,7 +2632,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) } /* We only support the AArch64 table format at present */ - switch (reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) { + switch (FIELD_GET(IDR0_TTF, reg)) { case IDR0_TTF_AARCH32_64: smmu->ias = 40; /* Fallthrough */ @@ -2674,22 +2655,22 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) } /* Queue sizes, capped at 4k */ - smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT, - reg >> IDR1_CMDQ_SHIFT & IDR1_CMDQ_MASK); + smmu->cmdq.q.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, + FIELD_GET(IDR1_CMDQS, reg)); if (!smmu->cmdq.q.max_n_shift) { /* Odd alignment restrictions on the base, so ignore for now */ dev_err(smmu->dev, "unit-length command queue not supported\n"); return -ENXIO; } - smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT, - reg >> IDR1_EVTQ_SHIFT & IDR1_EVTQ_MASK); - smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT, - reg >> IDR1_PRIQ_SHIFT & IDR1_PRIQ_MASK); + smmu->evtq.q.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT, + FIELD_GET(IDR1_EVTQS, reg)); + smmu->priq.q.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT, + FIELD_GET(IDR1_PRIQS, reg)); /* SID/SSID sizes */ - smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK; - smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK; + smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg); + smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg); /* * If the SMMU supports fewer bits than would fill a single L2 stream @@ -2702,8 +2683,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); /* Maximum number of outstanding stalls */ - smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT - & IDR5_STALL_MAX_MASK; + smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg); /* Page sizes */ if (reg & IDR5_GRAN64K) @@ -2719,7 +2699,7 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; /* Output address size */ - switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) { + switch (FIELD_GET(IDR5_OAS, reg)) { case IDR5_OAS_32_BIT: smmu->oas = 32; break; From ba08bdcbf7fd6c3b3a94496ca15b5b66dbea5034 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 26 Mar 2018 13:35:11 +0100 Subject: [PATCH 24/53] iommu/arm-smmu-v3: Clean up table definitions As with registers, use GENMASK and the bitfield accessors consistently for table fields, to save some lines and ease maintenance a little. This also catches a subtle off-by-one wherein bit 5 of CD.T0SZ was missing. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 147 ++++++++++++++---------------------- 1 file changed, 58 insertions(+), 89 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 48531c694c60..4539a6cde9f5 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -207,54 +207,46 @@ #define STRTAB_SPLIT 8 #define STRTAB_L1_DESC_DWORDS 1 -#define STRTAB_L1_DESC_SPAN_SHIFT 0 -#define STRTAB_L1_DESC_SPAN_MASK 0x1fUL +#define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0) #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(47, 6) #define STRTAB_STE_DWORDS 8 #define STRTAB_STE_0_V (1UL << 0) -#define STRTAB_STE_0_CFG_SHIFT 1 -#define STRTAB_STE_0_CFG_MASK 0x7UL -#define STRTAB_STE_0_CFG_ABORT (0UL << STRTAB_STE_0_CFG_SHIFT) -#define STRTAB_STE_0_CFG_BYPASS (4UL << STRTAB_STE_0_CFG_SHIFT) -#define STRTAB_STE_0_CFG_S1_TRANS (5UL << STRTAB_STE_0_CFG_SHIFT) -#define STRTAB_STE_0_CFG_S2_TRANS (6UL << STRTAB_STE_0_CFG_SHIFT) +#define STRTAB_STE_0_CFG GENMASK_ULL(3, 1) +#define STRTAB_STE_0_CFG_ABORT 0 +#define STRTAB_STE_0_CFG_BYPASS 4 +#define STRTAB_STE_0_CFG_S1_TRANS 5 +#define STRTAB_STE_0_CFG_S2_TRANS 6 -#define STRTAB_STE_0_S1FMT_SHIFT 4 -#define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT) +#define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4) +#define STRTAB_STE_0_S1FMT_LINEAR 0 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(47, 6) -#define STRTAB_STE_0_S1CDMAX_SHIFT 59 -#define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL +#define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59) #define STRTAB_STE_1_S1C_CACHE_NC 0UL #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL #define STRTAB_STE_1_S1C_CACHE_WT 2UL #define STRTAB_STE_1_S1C_CACHE_WB 3UL -#define STRTAB_STE_1_S1C_SH_NSH 0UL -#define STRTAB_STE_1_S1C_SH_OSH 2UL -#define STRTAB_STE_1_S1C_SH_ISH 3UL -#define STRTAB_STE_1_S1CIR_SHIFT 2 -#define STRTAB_STE_1_S1COR_SHIFT 4 -#define STRTAB_STE_1_S1CSH_SHIFT 6 +#define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2) +#define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4) +#define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6) #define STRTAB_STE_1_S1STALLD (1UL << 27) +#define STRTAB_STE_1_EATS GENMASK_ULL(29, 28) #define STRTAB_STE_1_EATS_ABT 0UL #define STRTAB_STE_1_EATS_TRANS 1UL #define STRTAB_STE_1_EATS_S1CHK 2UL -#define STRTAB_STE_1_EATS_SHIFT 28 +#define STRTAB_STE_1_STRW GENMASK_ULL(31, 30) #define STRTAB_STE_1_STRW_NSEL1 0UL #define STRTAB_STE_1_STRW_EL2 2UL -#define STRTAB_STE_1_STRW_SHIFT 30 +#define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44) #define STRTAB_STE_1_SHCFG_INCOMING 1UL -#define STRTAB_STE_1_SHCFG_SHIFT 44 -#define STRTAB_STE_2_S2VMID_SHIFT 0 -#define STRTAB_STE_2_S2VMID_MASK 0xffffUL -#define STRTAB_STE_2_VTCR_SHIFT 32 -#define STRTAB_STE_2_VTCR_MASK 0x7ffffUL +#define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0) +#define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32) #define STRTAB_STE_2_S2AA64 (1UL << 51) #define STRTAB_STE_2_S2ENDI (1UL << 52) #define STRTAB_STE_2_S2PTW (1UL << 54) @@ -264,56 +256,41 @@ /* Context descriptor (stage-1 only) */ #define CTXDESC_CD_DWORDS 8 -#define CTXDESC_CD_0_TCR_T0SZ_SHIFT 0 -#define ARM64_TCR_T0SZ_SHIFT 0 -#define ARM64_TCR_T0SZ_MASK 0x1fUL -#define CTXDESC_CD_0_TCR_TG0_SHIFT 6 -#define ARM64_TCR_TG0_SHIFT 14 -#define ARM64_TCR_TG0_MASK 0x3UL -#define CTXDESC_CD_0_TCR_IRGN0_SHIFT 8 -#define ARM64_TCR_IRGN0_SHIFT 8 -#define ARM64_TCR_IRGN0_MASK 0x3UL -#define CTXDESC_CD_0_TCR_ORGN0_SHIFT 10 -#define ARM64_TCR_ORGN0_SHIFT 10 -#define ARM64_TCR_ORGN0_MASK 0x3UL -#define CTXDESC_CD_0_TCR_SH0_SHIFT 12 -#define ARM64_TCR_SH0_SHIFT 12 -#define ARM64_TCR_SH0_MASK 0x3UL -#define CTXDESC_CD_0_TCR_EPD0_SHIFT 14 -#define ARM64_TCR_EPD0_SHIFT 7 -#define ARM64_TCR_EPD0_MASK 0x1UL -#define CTXDESC_CD_0_TCR_EPD1_SHIFT 30 -#define ARM64_TCR_EPD1_SHIFT 23 -#define ARM64_TCR_EPD1_MASK 0x1UL +#define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0) +#define ARM64_TCR_T0SZ GENMASK_ULL(5, 0) +#define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6) +#define ARM64_TCR_TG0 GENMASK_ULL(15, 14) +#define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8) +#define ARM64_TCR_IRGN0 GENMASK_ULL(9, 8) +#define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10) +#define ARM64_TCR_ORGN0 GENMASK_ULL(11, 10) +#define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12) +#define ARM64_TCR_SH0 GENMASK_ULL(13, 12) +#define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14) +#define ARM64_TCR_EPD0 (1ULL << 7) +#define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30) +#define ARM64_TCR_EPD1 (1ULL << 23) #define CTXDESC_CD_0_ENDI (1UL << 15) #define CTXDESC_CD_0_V (1UL << 31) -#define CTXDESC_CD_0_TCR_IPS_SHIFT 32 -#define ARM64_TCR_IPS_SHIFT 32 -#define ARM64_TCR_IPS_MASK 0x7UL -#define CTXDESC_CD_0_TCR_TBI0_SHIFT 38 -#define ARM64_TCR_TBI0_SHIFT 37 -#define ARM64_TCR_TBI0_MASK 0x1UL +#define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32) +#define ARM64_TCR_IPS GENMASK_ULL(34, 32) +#define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38) +#define ARM64_TCR_TBI0 (1ULL << 37) #define CTXDESC_CD_0_AA64 (1UL << 41) #define CTXDESC_CD_0_S (1UL << 44) #define CTXDESC_CD_0_R (1UL << 45) #define CTXDESC_CD_0_A (1UL << 46) -#define CTXDESC_CD_0_ASET_SHIFT 47 -#define CTXDESC_CD_0_ASET_SHARED (0UL << CTXDESC_CD_0_ASET_SHIFT) -#define CTXDESC_CD_0_ASET_PRIVATE (1UL << CTXDESC_CD_0_ASET_SHIFT) -#define CTXDESC_CD_0_ASID_SHIFT 48 -#define CTXDESC_CD_0_ASID_MASK 0xffffUL +#define CTXDESC_CD_0_ASET (1UL << 47) +#define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48) #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(47, 4) -#define CTXDESC_CD_3_MAIR_SHIFT 0 - /* Convert between AArch64 (CPU) TCR format and SMMU CD format */ -#define ARM_SMMU_TCR2CD(tcr, fld) \ - (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK) \ - << CTXDESC_CD_0_TCR_##fld##_SHIFT) +#define ARM_SMMU_TCR2CD(tcr, fld) FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \ + FIELD_GET(ARM64_TCR_##fld, tcr)) /* Command queue */ #define CMDQ_ENT_DWORDS 2 @@ -1060,8 +1037,8 @@ static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu, #ifdef __BIG_ENDIAN CTXDESC_CD_0_ENDI | #endif - CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET_PRIVATE | - CTXDESC_CD_0_AA64 | (u64)cfg->cd.asid << CTXDESC_CD_0_ASID_SHIFT | + CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET | + CTXDESC_CD_0_AA64 | FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) | CTXDESC_CD_0_V; /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */ @@ -1073,7 +1050,7 @@ static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu, val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK; cfg->cdptr[1] = cpu_to_le64(val); - cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT); + cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair); } /* Stream table manipulation functions */ @@ -1082,8 +1059,7 @@ arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) { u64 val = 0; - val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK) - << STRTAB_L1_DESC_SPAN_SHIFT; + val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span); val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK; *dst = cpu_to_le64(val); @@ -1132,10 +1108,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, }; if (val & STRTAB_STE_0_V) { - u64 cfg; - - cfg = val & STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT; - switch (cfg) { + switch (FIELD_GET(STRTAB_STE_0_CFG, val)) { case STRTAB_STE_0_CFG_BYPASS: break; case STRTAB_STE_0_CFG_S1_TRANS: @@ -1156,13 +1129,13 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, /* Bypass/fault */ if (!ste->assigned || !(ste->s1_cfg || ste->s2_cfg)) { if (!ste->assigned && disable_bypass) - val |= STRTAB_STE_0_CFG_ABORT; + val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT); else - val |= STRTAB_STE_0_CFG_BYPASS; + val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS); dst[0] = cpu_to_le64(val); - dst[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING - << STRTAB_STE_1_SHCFG_SHIFT); + dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG, + STRTAB_STE_1_SHCFG_INCOMING)); dst[2] = 0; /* Nuke the VMID */ /* * The SMMU can perform negative caching, so we must sync @@ -1176,40 +1149,36 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, if (ste->s1_cfg) { BUG_ON(ste_live); dst[1] = cpu_to_le64( - STRTAB_STE_1_S1C_CACHE_WBRA - << STRTAB_STE_1_S1CIR_SHIFT | - STRTAB_STE_1_S1C_CACHE_WBRA - << STRTAB_STE_1_S1COR_SHIFT | - STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT | + FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) | + FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) | #ifdef CONFIG_PCI_ATS - STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT | + FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS) | #endif - STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT); + FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1)); if (smmu->features & ARM_SMMU_FEAT_STALLS && !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) | - STRTAB_STE_0_CFG_S1_TRANS; + FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS); } if (ste->s2_cfg) { BUG_ON(ste_live); dst[2] = cpu_to_le64( - ste->s2_cfg->vmid << STRTAB_STE_2_S2VMID_SHIFT | - (ste->s2_cfg->vtcr & STRTAB_STE_2_VTCR_MASK) - << STRTAB_STE_2_VTCR_SHIFT | + FIELD_PREP(STRTAB_STE_2_S2VMID, ste->s2_cfg->vmid) | + FIELD_PREP(STRTAB_STE_2_VTCR, ste->s2_cfg->vtcr) | #ifdef __BIG_ENDIAN STRTAB_STE_2_S2ENDI | #endif STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2R); - dst[3] = cpu_to_le64(ste->s2_cfg->vttbr & - STRTAB_STE_3_S2TTB_MASK); + dst[3] = cpu_to_le64(ste->s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK); - val |= STRTAB_STE_0_CFG_S2_TRANS; + val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS); } arm_smmu_sync_ste_for_sid(smmu, sid); From 7417b99c49e5bb77e04d64c915da2ee4bfcbf8a8 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 26 Mar 2018 13:35:12 +0100 Subject: [PATCH 25/53] iommu/arm-smmu-v3: Clean up queue definitions As with registers and tables, use GENMASK and the bitfield accessors consistently for queue fields, to save some lines and ease maintenance a little. This now leaves everything in a nice state where all named field definitions expect to be used with bitfield accessors (although since single-bit fields can still be used directly we leave some of those uses as-is to avoid unnecessary churn), while the few remaining *_MASK definitions apply exclusively to in-place values. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 126 ++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 72 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 4539a6cde9f5..ae6049bddd73 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -184,6 +184,7 @@ #define ARM_SMMU_SH_OSH 2 #define ARM_SMMU_SH_ISH 3 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1 +#define ARM_SMMU_MEMATTR_OIWB 0xf #define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1)) #define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift)) @@ -301,64 +302,49 @@ #define CMDQ_ERR_CERROR_ILL_IDX 1 #define CMDQ_ERR_CERROR_ABT_IDX 2 -#define CMDQ_0_OP_SHIFT 0 -#define CMDQ_0_OP_MASK 0xffUL +#define CMDQ_0_OP GENMASK_ULL(7, 0) #define CMDQ_0_SSV (1UL << 11) -#define CMDQ_PREFETCH_0_SID_SHIFT 32 -#define CMDQ_PREFETCH_1_SIZE_SHIFT 0 +#define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32) +#define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0) #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12) -#define CMDQ_CFGI_0_SID_SHIFT 32 -#define CMDQ_CFGI_0_SID_MASK 0xffffffffUL +#define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32) #define CMDQ_CFGI_1_LEAF (1UL << 0) -#define CMDQ_CFGI_1_RANGE_SHIFT 0 -#define CMDQ_CFGI_1_RANGE_MASK 0x1fUL +#define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0) -#define CMDQ_TLBI_0_VMID_SHIFT 32 -#define CMDQ_TLBI_0_ASID_SHIFT 48 +#define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32) +#define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48) #define CMDQ_TLBI_1_LEAF (1UL << 0) #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12) #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(47, 12) -#define CMDQ_PRI_0_SSID_SHIFT 12 -#define CMDQ_PRI_0_SSID_MASK 0xfffffUL -#define CMDQ_PRI_0_SID_SHIFT 32 -#define CMDQ_PRI_0_SID_MASK 0xffffffffUL -#define CMDQ_PRI_1_GRPID_SHIFT 0 -#define CMDQ_PRI_1_GRPID_MASK 0x1ffUL -#define CMDQ_PRI_1_RESP_SHIFT 12 -#define CMDQ_PRI_1_RESP_DENY (0UL << CMDQ_PRI_1_RESP_SHIFT) -#define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT) -#define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT) +#define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12) +#define CMDQ_PRI_0_SID GENMASK_ULL(63, 32) +#define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0) +#define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12) -#define CMDQ_SYNC_0_CS_SHIFT 12 -#define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT) -#define CMDQ_SYNC_0_CS_IRQ (1UL << CMDQ_SYNC_0_CS_SHIFT) -#define CMDQ_SYNC_0_CS_SEV (2UL << CMDQ_SYNC_0_CS_SHIFT) -#define CMDQ_SYNC_0_MSH_SHIFT 22 -#define CMDQ_SYNC_0_MSH_ISH (3UL << CMDQ_SYNC_0_MSH_SHIFT) -#define CMDQ_SYNC_0_MSIATTR_SHIFT 24 -#define CMDQ_SYNC_0_MSIATTR_OIWB (0xfUL << CMDQ_SYNC_0_MSIATTR_SHIFT) -#define CMDQ_SYNC_0_MSIDATA_SHIFT 32 -#define CMDQ_SYNC_0_MSIDATA_MASK 0xffffffffUL +#define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12) +#define CMDQ_SYNC_0_CS_NONE 0 +#define CMDQ_SYNC_0_CS_IRQ 1 +#define CMDQ_SYNC_0_CS_SEV 2 +#define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22) +#define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24) +#define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32) #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(47, 2) /* Event queue */ #define EVTQ_ENT_DWORDS 4 #define EVTQ_MAX_SZ_SHIFT 7 -#define EVTQ_0_ID_SHIFT 0 -#define EVTQ_0_ID_MASK 0xffUL +#define EVTQ_0_ID GENMASK_ULL(7, 0) /* PRI queue */ #define PRIQ_ENT_DWORDS 2 #define PRIQ_MAX_SZ_SHIFT 8 -#define PRIQ_0_SID_SHIFT 0 -#define PRIQ_0_SID_MASK 0xffffffffUL -#define PRIQ_0_SSID_SHIFT 32 -#define PRIQ_0_SSID_MASK 0xfffffUL +#define PRIQ_0_SID GENMASK_ULL(31, 0) +#define PRIQ_0_SSID GENMASK_ULL(51, 32) #define PRIQ_0_PERM_PRIV (1UL << 58) #define PRIQ_0_PERM_EXEC (1UL << 59) #define PRIQ_0_PERM_READ (1UL << 60) @@ -366,8 +352,7 @@ #define PRIQ_0_PRG_LAST (1UL << 62) #define PRIQ_0_SSID_V (1UL << 63) -#define PRIQ_1_PRG_IDX_SHIFT 0 -#define PRIQ_1_PRG_IDX_MASK 0x1ffUL +#define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0) #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12) /* High-level queue structures */ @@ -384,9 +369,9 @@ MODULE_PARM_DESC(disable_bypass, "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); enum pri_resp { - PRI_RESP_DENY, - PRI_RESP_FAIL, - PRI_RESP_SUCC, + PRI_RESP_DENY = 0, + PRI_RESP_FAIL = 1, + PRI_RESP_SUCC = 2, }; enum arm_smmu_msi_index { @@ -790,67 +775,64 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent) static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) { memset(cmd, 0, CMDQ_ENT_DWORDS << 3); - cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT; + cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode); switch (ent->opcode) { case CMDQ_OP_TLBI_EL2_ALL: case CMDQ_OP_TLBI_NSNH_ALL: break; case CMDQ_OP_PREFETCH_CFG: - cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT; - cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT; + cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid); + cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size); cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK; break; case CMDQ_OP_CFGI_STE: - cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT; - cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0; + cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid); + cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf); break; case CMDQ_OP_CFGI_ALL: /* Cover the entire SID range */ - cmd[1] |= CMDQ_CFGI_1_RANGE_MASK << CMDQ_CFGI_1_RANGE_SHIFT; + cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31); break; case CMDQ_OP_TLBI_NH_VA: - cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT; - cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0; + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); + cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK; break; case CMDQ_OP_TLBI_S2_IPA: - cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT; - cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0; + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); + cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf); cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK; break; case CMDQ_OP_TLBI_NH_ASID: - cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT; + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid); /* Fallthrough */ case CMDQ_OP_TLBI_S12_VMALL: - cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT; + cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid); break; case CMDQ_OP_PRI_RESP: - cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0; - cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT; - cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT; - cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT; + cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid); + cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid); + cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid); + cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid); switch (ent->pri.resp) { case PRI_RESP_DENY: - cmd[1] |= CMDQ_PRI_1_RESP_DENY; - break; case PRI_RESP_FAIL: - cmd[1] |= CMDQ_PRI_1_RESP_FAIL; - break; case PRI_RESP_SUCC: - cmd[1] |= CMDQ_PRI_1_RESP_SUCC; break; default: return -EINVAL; } + cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp); break; case CMDQ_OP_CMD_SYNC: if (ent->sync.msiaddr) - cmd[0] |= CMDQ_SYNC_0_CS_IRQ; + cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ); else - cmd[0] |= CMDQ_SYNC_0_CS_SEV; - cmd[0] |= CMDQ_SYNC_0_MSH_ISH | CMDQ_SYNC_0_MSIATTR_OIWB; - cmd[0] |= (u64)ent->sync.msidata << CMDQ_SYNC_0_MSIDATA_SHIFT; + cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV); + cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH); + cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB); + cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIDATA, ent->sync.msidata); cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK; break; default: @@ -1239,7 +1221,7 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev) do { while (!queue_remove_raw(q, evt)) { - u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK; + u8 id = FIELD_GET(EVTQ_0_ID, evt[0]); dev_info(smmu->dev, "event 0x%02x received:\n", id); for (i = 0; i < ARRAY_SIZE(evt); ++i) @@ -1267,11 +1249,11 @@ static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) u16 grpid; bool ssv, last; - sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK; - ssv = evt[0] & PRIQ_0_SSID_V; - ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0; - last = evt[0] & PRIQ_0_PRG_LAST; - grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK; + sid = FIELD_GET(PRIQ_0_SID, evt[0]); + ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]); + ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0; + last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]); + grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]); dev_info(smmu->dev, "unexpected PRI request received:\n"); dev_info(smmu->dev, From 6c89928ff7a0f7b4ebc2ac1acab9f99d5f50ed75 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 26 Mar 2018 13:35:13 +0100 Subject: [PATCH 26/53] iommu/io-pgtable-arm: Support 52-bit physical address Bring io-pgtable-arm in line with the ARMv8.2-LPA feature allowing 52-bit physical addresses when using the 64KB translation granule. This will be supported by SMMUv3.1. Tested-by: Nate Watterson Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm.c | 67 +++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 51e5c43caed1..a5be4c92c5c8 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -21,6 +21,7 @@ #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt #include +#include #include #include #include @@ -32,7 +33,7 @@ #include "io-pgtable.h" -#define ARM_LPAE_MAX_ADDR_BITS 48 +#define ARM_LPAE_MAX_ADDR_BITS 52 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 #define ARM_LPAE_MAX_LEVELS 4 @@ -86,6 +87,8 @@ #define ARM_LPAE_PTE_TYPE_TABLE 3 #define ARM_LPAE_PTE_TYPE_PAGE 3 +#define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12) + #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) @@ -159,6 +162,7 @@ #define ARM_LPAE_TCR_PS_42_BIT 0x3ULL #define ARM_LPAE_TCR_PS_44_BIT 0x4ULL #define ARM_LPAE_TCR_PS_48_BIT 0x5ULL +#define ARM_LPAE_TCR_PS_52_BIT 0x6ULL #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) #define ARM_LPAE_MAIR_ATTR_MASK 0xff @@ -170,9 +174,7 @@ #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 /* IOPTE accessors */ -#define iopte_deref(pte,d) \ - (__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \ - & ~(ARM_LPAE_GRANULE(d) - 1ULL))) +#define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) #define iopte_type(pte,l) \ (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) @@ -184,12 +186,6 @@ (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) : \ (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK)) -#define iopte_to_pfn(pte,d) \ - (((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) >> (d)->pg_shift) - -#define pfn_to_iopte(pfn,d) \ - (((pfn) << (d)->pg_shift) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) - struct arm_lpae_io_pgtable { struct io_pgtable iop; @@ -203,6 +199,27 @@ struct arm_lpae_io_pgtable { typedef u64 arm_lpae_iopte; +static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, + struct arm_lpae_io_pgtable *data) +{ + arm_lpae_iopte pte = paddr; + + /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */ + return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; +} + +static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte, + struct arm_lpae_io_pgtable *data) +{ + phys_addr_t paddr = pte & ARM_LPAE_PTE_ADDR_MASK; + + if (data->pg_shift < 16) + return paddr; + + /* Rotate the packed high-order bits back to the top */ + return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); +} + static bool selftest_running = false; static dma_addr_t __arm_lpae_dma_addr(void *pages) @@ -287,7 +304,7 @@ static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, pte |= ARM_LPAE_PTE_TYPE_BLOCK; pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; - pte |= pfn_to_iopte(paddr >> data->pg_shift, data); + pte |= paddr_to_iopte(paddr, data); __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); } @@ -528,7 +545,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, if (size == split_sz) unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data); - blk_paddr = iopte_to_pfn(blk_pte, data) << data->pg_shift; + blk_paddr = iopte_to_paddr(blk_pte, data); pte = iopte_prot(blk_pte); for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { @@ -652,12 +669,13 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, found_translation: iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); - return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; + return iopte_to_paddr(pte, data) | iova; } static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) { - unsigned long granule; + unsigned long granule, page_sizes; + unsigned int max_addr_bits = 48; /* * We need to restrict the supported page sizes to match the @@ -677,17 +695,24 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) switch (granule) { case SZ_4K: - cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); + page_sizes = (SZ_4K | SZ_2M | SZ_1G); break; case SZ_16K: - cfg->pgsize_bitmap &= (SZ_16K | SZ_32M); + page_sizes = (SZ_16K | SZ_32M); break; case SZ_64K: - cfg->pgsize_bitmap &= (SZ_64K | SZ_512M); + max_addr_bits = 52; + page_sizes = (SZ_64K | SZ_512M); + if (cfg->oas > 48) + page_sizes |= 1ULL << 42; /* 4TB */ break; default: - cfg->pgsize_bitmap = 0; + page_sizes = 0; } + + cfg->pgsize_bitmap &= page_sizes; + cfg->ias = min(cfg->ias, max_addr_bits); + cfg->oas = min(cfg->oas, max_addr_bits); } static struct arm_lpae_io_pgtable * @@ -784,6 +809,9 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) case 48: reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT); break; + case 52: + reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_IPS_SHIFT); + break; default: goto out_free_data; } @@ -891,6 +919,9 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) case 48: reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT); break; + case 52: + reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_PS_SHIFT); + break; default: goto out_free_data; } From 6619c9138533597dc0ff52155d263f14e8578ca2 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 26 Mar 2018 13:35:14 +0100 Subject: [PATCH 27/53] iommu/arm-smmu-v3: Support 52-bit physical address Implement SMMUv3.1 support for 52-bit physical addresses. Since a 52-bit OAS implies 64KB translation granule support, permitting level 1 block entries there is simple, and the rest is just extending address fields. Tested-by: Nate Watterson Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index ae6049bddd73..b73a5950eb65 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -91,6 +91,7 @@ #define IDR5_OAS_42_BIT 3 #define IDR5_OAS_44_BIT 4 #define IDR5_OAS_48_BIT 5 +#define IDR5_OAS_52_BIT 6 #define ARM_SMMU_CR0 0x20 #define CR0_CMDQEN (1 << 3) @@ -147,7 +148,7 @@ #define ARM_SMMU_STRTAB_BASE 0x80 #define STRTAB_BASE_RA (1UL << 62) -#define STRTAB_BASE_ADDR_MASK GENMASK_ULL(47, 6) +#define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6) #define ARM_SMMU_STRTAB_BASE_CFG 0x88 #define STRTAB_BASE_CFG_FMT GENMASK(17, 16) @@ -175,7 +176,7 @@ #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc /* Common MSI config fields */ -#define MSI_CFG0_ADDR_MASK GENMASK_ULL(47, 2) +#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2) #define MSI_CFG2_SH GENMASK(5, 4) #define MSI_CFG2_MEMATTR GENMASK(3, 0) @@ -194,7 +195,7 @@ Q_IDX(q, p) * (q)->ent_dwords) #define Q_BASE_RWA (1UL << 62) -#define Q_BASE_ADDR_MASK GENMASK_ULL(47, 5) +#define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5) #define Q_BASE_LOG2SIZE GENMASK(4, 0) /* @@ -209,7 +210,7 @@ #define STRTAB_L1_DESC_DWORDS 1 #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0) -#define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(47, 6) +#define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6) #define STRTAB_STE_DWORDS 8 #define STRTAB_STE_0_V (1UL << 0) @@ -221,7 +222,7 @@ #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4) #define STRTAB_STE_0_S1FMT_LINEAR 0 -#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(47, 6) +#define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6) #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59) #define STRTAB_STE_1_S1C_CACHE_NC 0UL @@ -253,7 +254,7 @@ #define STRTAB_STE_2_S2PTW (1UL << 54) #define STRTAB_STE_2_S2R (1UL << 58) -#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(47, 4) +#define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4) /* Context descriptor (stage-1 only) */ #define CTXDESC_CD_DWORDS 8 @@ -287,7 +288,7 @@ #define CTXDESC_CD_0_ASET (1UL << 47) #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48) -#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(47, 4) +#define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4) /* Convert between AArch64 (CPU) TCR format and SMMU CD format */ #define ARM_SMMU_TCR2CD(tcr, fld) FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \ @@ -317,7 +318,7 @@ #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48) #define CMDQ_TLBI_1_LEAF (1UL << 0) #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12) -#define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(47, 12) +#define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12) #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12) #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32) @@ -331,7 +332,7 @@ #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22) #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24) #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32) -#define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(47, 2) +#define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2) /* Event queue */ #define EVTQ_ENT_DWORDS 4 @@ -1622,7 +1623,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) return -ENOMEM; domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - domain->geometry.aperture_end = (1UL << ias) - 1; + domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1; domain->geometry.force_aperture = true; ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg); @@ -2644,11 +2645,6 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) if (reg & IDR5_GRAN4K) smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; - if (arm_smmu_ops.pgsize_bitmap == -1UL) - arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; - else - arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; - /* Output address size */ switch (FIELD_GET(IDR5_OAS, reg)) { case IDR5_OAS_32_BIT: @@ -2666,6 +2662,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) case IDR5_OAS_44_BIT: smmu->oas = 44; break; + case IDR5_OAS_52_BIT: + smmu->oas = 52; + smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */ + break; default: dev_info(smmu->dev, "unknown output address size. Truncating to 48-bit\n"); @@ -2674,6 +2674,11 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) smmu->oas = 48; } + if (arm_smmu_ops.pgsize_bitmap == -1UL) + arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; + else + arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; + /* Set the DMA mask for our table walker */ if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas))) dev_warn(smmu->dev, From dcd189e6d2eca4663da6120463fbff0995bc06eb Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 26 Mar 2018 13:35:15 +0100 Subject: [PATCH 28/53] iommu/arm-smmu-v3: Support 52-bit virtual address Stage 1 input addresses are effectively 64-bit in SMMUv3 anyway, so really all that's involved is letting io-pgtable know the appropriate upper bound for T0SZ. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index b73a5950eb65..1d647104bccc 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -92,6 +92,8 @@ #define IDR5_OAS_44_BIT 4 #define IDR5_OAS_48_BIT 5 #define IDR5_OAS_52_BIT 6 +#define IDR5_VAX GENMASK(11, 10) +#define IDR5_VAX_52_BIT 1 #define ARM_SMMU_CR0 0x20 #define CR0_CMDQEN (1 << 3) @@ -551,6 +553,7 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_STALLS (1 << 11) #define ARM_SMMU_FEAT_HYP (1 << 12) #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13) +#define ARM_SMMU_FEAT_VAX (1 << 14) u32 features; #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0) @@ -1591,7 +1594,8 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: - ias = VA_BITS; + ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48; + ias = min_t(unsigned long, ias, VA_BITS); oas = smmu->ias; fmt = ARM_64_LPAE_S1; finalise_stage_fn = arm_smmu_domain_finalise_s1; @@ -2645,6 +2649,10 @@ static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) if (reg & IDR5_GRAN4K) smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; + /* Input address size */ + if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT) + smmu->features |= ARM_SMMU_FEAT_VAX; + /* Output address size */ switch (FIELD_GET(IDR5_OAS, reg)) { case IDR5_OAS_32_BIT: From f746a025d35f6c73e6021d0e37166e048cda36ed Mon Sep 17 00:00:00 2001 From: Shaokun Zhang Date: Thu, 22 Mar 2018 18:18:06 +0800 Subject: [PATCH 29/53] iommu/vt-d:Remove unused variable Unused after commit <42e8c186b595> ("iommu/vt-d: Simplify io/tlb flushing in intel_iommu_unmap"), cleanup it. Cc: David Woodhouse Cc: Joerg Roedel Signed-off-by: Shaokun Zhang Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 582fd01cb7d1..d49e0d3ab748 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -5072,7 +5072,6 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, { struct dmar_domain *dmar_domain = to_dmar_domain(domain); struct page *freelist = NULL; - struct intel_iommu *iommu; unsigned long start_pfn, last_pfn; unsigned int npages; int iommu_id, level = 0; @@ -5091,12 +5090,9 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, npages = last_pfn - start_pfn + 1; - for_each_domain_iommu(iommu_id, dmar_domain) { - iommu = g_iommus[iommu_id]; - + for_each_domain_iommu(iommu_id, dmar_domain) iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain, start_pfn, npages, !freelist, 0); - } dma_free_pagelist(freelist); From 39ffe39545cd5cb5b8cee9f0469165cf24dc62c2 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:33 +0100 Subject: [PATCH 30/53] iommu/amd: Take into account that alloc_dev_data() may return NULL find_dev_data() does not check whether the return value alloc_dev_data() is NULL. This was okay once because the pointer was returned once as-is. Since commit df3f7a6e8e85 ("iommu/amd: Use is_attach_deferred call-back") the pointer may be used within find_dev_data() so a NULL check is required. Cc: Baoquan He Fixes: df3f7a6e8e85 ("iommu/amd: Use is_attach_deferred call-back") Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 4cd19f93ca15..121c54f1c768 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -310,6 +310,8 @@ static struct iommu_dev_data *find_dev_data(u16 devid) if (dev_data == NULL) { dev_data = alloc_dev_data(devid); + if (!dev_data) + return NULL; if (translation_pre_enabled(iommu)) dev_data->defer_attach = true; From 779da73273fc4c4c6f41579a95e4fb7880a1720e Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:34 +0100 Subject: [PATCH 31/53] iommu/amd: Turn dev_data_list into a lock less list alloc_dev_data() adds new items to dev_data_list and search_dev_data() is searching for items in this list. Both protect the access to the list with a spinlock. There is no need to navigate forth and back within the list and there is also no deleting of a specific item. This qualifies the list to become a lock less list and as part of this, the spinlock can be removed. With this change the ordering of those items within the list is changed: before the change new items were added to the end of the list, now they are added to the front. I don't think it matters but wanted to mention it. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 28 ++++++++++------------------ drivers/iommu/amd_iommu_types.h | 2 +- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 121c54f1c768..d4c2b1a11924 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -83,8 +83,7 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); /* List of all available dev_data structures */ -static LIST_HEAD(dev_data_list); -static DEFINE_SPINLOCK(dev_data_list_lock); +static LLIST_HEAD(dev_data_list); LIST_HEAD(ioapic_map); LIST_HEAD(hpet_map); @@ -203,40 +202,33 @@ static struct dma_ops_domain* to_dma_ops_domain(struct protection_domain *domain static struct iommu_dev_data *alloc_dev_data(u16 devid) { struct iommu_dev_data *dev_data; - unsigned long flags; dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); if (!dev_data) return NULL; dev_data->devid = devid; - - spin_lock_irqsave(&dev_data_list_lock, flags); - list_add_tail(&dev_data->dev_data_list, &dev_data_list); - spin_unlock_irqrestore(&dev_data_list_lock, flags); - ratelimit_default_init(&dev_data->rs); + llist_add(&dev_data->dev_data_list, &dev_data_list); return dev_data; } static struct iommu_dev_data *search_dev_data(u16 devid) { struct iommu_dev_data *dev_data; - unsigned long flags; + struct llist_node *node; - spin_lock_irqsave(&dev_data_list_lock, flags); - list_for_each_entry(dev_data, &dev_data_list, dev_data_list) { + if (llist_empty(&dev_data_list)) + return NULL; + + node = dev_data_list.first; + llist_for_each_entry(dev_data, node, dev_data_list) { if (dev_data->devid == devid) - goto out_unlock; + return dev_data; } - dev_data = NULL; - -out_unlock: - spin_unlock_irqrestore(&dev_data_list_lock, flags); - - return dev_data; + return NULL; } static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index da886b0095aa..1c9b080276c9 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -627,7 +627,7 @@ struct devid_map { */ struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ - struct list_head dev_data_list; /* For global dev_data_list */ + struct llist_node dev_data_list; /* For global dev_data_list */ struct protection_domain *domain; /* Domain the device is bound to */ u16 devid; /* PCI Device ID */ u16 alias; /* Alias Device ID */ From 2bc00180890427dcc092b2f2b0d03c904bcade29 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:35 +0100 Subject: [PATCH 32/53] iommu/amd: Split domain id out of amd_iommu_devtable_lock domain_id_alloc() and domain_id_free() is used for id management. Those two function share a bitmap (amd_iommu_pd_alloc_bitmap) and set/clear bits based on id allocation. There is no need to share this with amd_iommu_devtable_lock, it can use its own lock for this operation. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index d4c2b1a11924..fcfdce70707d 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -81,6 +81,7 @@ #define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38)) static DEFINE_RWLOCK(amd_iommu_devtable_lock); +static DEFINE_SPINLOCK(pd_bitmap_lock); /* List of all available dev_data structures */ static LLIST_HEAD(dev_data_list); @@ -1600,29 +1601,26 @@ static void del_domain_from_list(struct protection_domain *domain) static u16 domain_id_alloc(void) { - unsigned long flags; int id; - write_lock_irqsave(&amd_iommu_devtable_lock, flags); + spin_lock(&pd_bitmap_lock); id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID); BUG_ON(id == 0); if (id > 0 && id < MAX_DOMAIN_ID) __set_bit(id, amd_iommu_pd_alloc_bitmap); else id = 0; - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + spin_unlock(&pd_bitmap_lock); return id; } static void domain_id_free(int id) { - unsigned long flags; - - write_lock_irqsave(&amd_iommu_devtable_lock, flags); + spin_lock(&pd_bitmap_lock); if (id > 0 && id < MAX_DOMAIN_ID) __clear_bit(id, amd_iommu_pd_alloc_bitmap); - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + spin_unlock(&pd_bitmap_lock); } #define DEFINE_FREE_PT_FN(LVL, FN) \ From ea6166f4b83e9cfba1c18f46a764d50045682fe5 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:36 +0100 Subject: [PATCH 33/53] iommu/amd: Split irq_lookup_table out of the amd_iommu_devtable_lock The function get_irq_table() reads/writes irq_lookup_table while holding the amd_iommu_devtable_lock. It also modifies amd_iommu_dev_table[].data[2]. set_dte_entry() is using amd_iommu_dev_table[].data[0|1] (under the domain->lock) so it should be okay. The access to the iommu is serialized with its own (iommu's) lock. So split out get_irq_table() out of amd_iommu_devtable_lock's lock. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index fcfdce70707d..a87d2fee68db 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -82,6 +82,7 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); static DEFINE_SPINLOCK(pd_bitmap_lock); +static DEFINE_SPINLOCK(iommu_table_lock); /* List of all available dev_data structures */ static LLIST_HEAD(dev_data_list); @@ -3623,7 +3624,7 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, bool ioapic) unsigned long flags; u16 alias; - write_lock_irqsave(&amd_iommu_devtable_lock, flags); + spin_lock_irqsave(&iommu_table_lock, flags); iommu = amd_iommu_rlookup_table[devid]; if (!iommu) @@ -3688,7 +3689,7 @@ out: iommu_completion_wait(iommu); out_unlock: - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + spin_unlock_irqrestore(&iommu_table_lock, flags); return table; } From fde65dd3d3096e8f6ecc7bbe544eb91f4220772c Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:37 +0100 Subject: [PATCH 34/53] iommu/amd: Remove the special case from alloc_irq_table() alloc_irq_table() has a special ioapic argument. If set then it will pre-allocate / reserve the first 32 indexes. The argument is only once true and it would make alloc_irq_table() a little simpler if we would extract the special bits to the caller. The caller of irq_remapping_alloc() is holding irq_domain_mutex so the initialization of iommu->irte_ops->set_allocated() should not race against other user. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index a87d2fee68db..a5982a61f181 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3617,7 +3617,7 @@ static struct irq_remap_table *get_irq_table(u16 devid) return table; } -static struct irq_remap_table *alloc_irq_table(u16 devid, bool ioapic) +static struct irq_remap_table *alloc_irq_table(u16 devid) { struct irq_remap_table *table = NULL; struct amd_iommu *iommu; @@ -3651,10 +3651,6 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, bool ioapic) /* Initialize table spin-lock */ raw_spin_lock_init(&table->lock); - if (ioapic) - /* Keep the first 32 indexes free for IOAPIC interrupts */ - table->min_index = 32; - table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_ATOMIC); if (!table->table) { kfree(table); @@ -3669,12 +3665,6 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, bool ioapic) memset(table->table, 0, (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2))); - if (ioapic) { - int i; - - for (i = 0; i < 32; ++i) - iommu->irte_ops->set_allocated(table, i); - } irq_lookup_table[devid] = table; set_dte_irq_entry(devid, table); @@ -3704,7 +3694,7 @@ static int alloc_irq_index(u16 devid, int count, bool align) if (!iommu) return -ENODEV; - table = alloc_irq_table(devid, false); + table = alloc_irq_table(devid); if (!table) return -ENODEV; @@ -4130,10 +4120,26 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, return ret; if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { - if (alloc_irq_table(devid, true)) + struct irq_remap_table *table; + struct amd_iommu *iommu; + + table = alloc_irq_table(devid); + if (table) { + if (!table->min_index) { + /* + * Keep the first 32 indexes free for IOAPIC + * interrupts. + */ + table->min_index = 32; + iommu = amd_iommu_rlookup_table[devid]; + for (i = 0; i < 32; ++i) + iommu->irte_ops->set_allocated(table, i); + } + WARN_ON(table->min_index != 32); index = info->ioapic_pin; - else + } else { ret = -ENOMEM; + } } else { bool align = (info->type == X86_IRQ_ALLOC_TYPE_MSI); From 4fde541c9dc114c5b448ad34b0286fe8b7c550f1 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:38 +0100 Subject: [PATCH 35/53] iommu/amd: Use `table' instead `irt' as variable name in amd_iommu_update_ga() The variable of type struct irq_remap_table is always named `table' except in amd_iommu_update_ga() where it is called `irt'. Make it consistent and name it also `table'. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index a5982a61f181..ea120c7b46c9 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -4405,7 +4405,7 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) { unsigned long flags; struct amd_iommu *iommu; - struct irq_remap_table *irt; + struct irq_remap_table *table; struct amd_ir_data *ir_data = (struct amd_ir_data *)data; int devid = ir_data->irq_2_irte.devid; struct irte_ga *entry = (struct irte_ga *) ir_data->entry; @@ -4419,11 +4419,11 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) if (!iommu) return -ENODEV; - irt = get_irq_table(devid); - if (!irt) + table = get_irq_table(devid); + if (!table) return -ENODEV; - raw_spin_lock_irqsave(&irt->lock, flags); + raw_spin_lock_irqsave(&table->lock, flags); if (ref->lo.fields_vapic.guest_mode) { if (cpu >= 0) @@ -4432,7 +4432,7 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) barrier(); } - raw_spin_unlock_irqrestore(&irt->lock, flags); + raw_spin_unlock_irqrestore(&table->lock, flags); iommu_flush_irt(iommu, devid); iommu_completion_wait(iommu); From 2fcc1e8ac4a8514c64f946178fc36c2e30e56a41 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:39 +0100 Subject: [PATCH 36/53] iommu/amd: Factor out setting the remap table for a devid Setting the IRQ remap table for a specific devid (or its alias devid) includes three steps. Those three steps are always repeated each time this is done. Introduce a new helper function, move those steps there and use that function instead. The compiler can still decide if it is worth to inline. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index ea120c7b46c9..11ea2d656be8 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3617,6 +3617,14 @@ static struct irq_remap_table *get_irq_table(u16 devid) return table; } +static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid, + struct irq_remap_table *table) +{ + irq_lookup_table[devid] = table; + set_dte_irq_entry(devid, table); + iommu_flush_dte(iommu, devid); +} + static struct irq_remap_table *alloc_irq_table(u16 devid) { struct irq_remap_table *table = NULL; @@ -3637,9 +3645,7 @@ static struct irq_remap_table *alloc_irq_table(u16 devid) alias = amd_iommu_alias_table[devid]; table = irq_lookup_table[alias]; if (table) { - irq_lookup_table[devid] = table; - set_dte_irq_entry(devid, table); - iommu_flush_dte(iommu, devid); + set_remap_table_entry(iommu, devid, table); goto out; } @@ -3666,14 +3672,9 @@ static struct irq_remap_table *alloc_irq_table(u16 devid) (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2))); - irq_lookup_table[devid] = table; - set_dte_irq_entry(devid, table); - iommu_flush_dte(iommu, devid); - if (devid != alias) { - irq_lookup_table[alias] = table; - set_dte_irq_entry(alias, table); - iommu_flush_dte(iommu, alias); - } + set_remap_table_entry(iommu, devid, table); + if (devid != alias) + set_remap_table_entry(iommu, alias, table); out: iommu_completion_wait(iommu); From 993ca6e063a69a0c65ca42ed449b6bc1b3844151 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:40 +0100 Subject: [PATCH 37/53] iommu/amd: Drop the lock while allocating new irq remap table The irq_remap_table is allocated while the iommu_table_lock is held with interrupts disabled. >From looking at the call sites, all callers are in the early device initialisation (apic_bsp_setup(), pci_enable_device(), pci_enable_msi()) so make sense to drop the lock which also enables interrupts and try to allocate that memory with GFP_KERNEL instead GFP_ATOMIC. Since during the allocation the iommu_table_lock is dropped, we need to recheck if table exists after the lock has been reacquired. I *think* that it is impossible that the "devid" entry appears in irq_lookup_table while the lock is dropped since the same device can only be probed once. However I check for both cases, just to be sure. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 65 +++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 11ea2d656be8..c493d345b3ef 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3617,6 +3617,30 @@ static struct irq_remap_table *get_irq_table(u16 devid) return table; } +static struct irq_remap_table *__alloc_irq_table(void) +{ + struct irq_remap_table *table; + + table = kzalloc(sizeof(*table), GFP_KERNEL); + if (!table) + return NULL; + + table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_KERNEL); + if (!table->table) { + kfree(table); + return NULL; + } + raw_spin_lock_init(&table->lock); + + if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) + memset(table->table, 0, + MAX_IRQS_PER_TABLE * sizeof(u32)); + else + memset(table->table, 0, + (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2))); + return table; +} + static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid, struct irq_remap_table *table) { @@ -3628,6 +3652,7 @@ static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid, static struct irq_remap_table *alloc_irq_table(u16 devid) { struct irq_remap_table *table = NULL; + struct irq_remap_table *new_table = NULL; struct amd_iommu *iommu; unsigned long flags; u16 alias; @@ -3646,42 +3671,44 @@ static struct irq_remap_table *alloc_irq_table(u16 devid) table = irq_lookup_table[alias]; if (table) { set_remap_table_entry(iommu, devid, table); - goto out; + goto out_wait; } + spin_unlock_irqrestore(&iommu_table_lock, flags); /* Nothing there yet, allocate new irq remapping table */ - table = kzalloc(sizeof(*table), GFP_ATOMIC); - if (!table) + new_table = __alloc_irq_table(); + if (!new_table) + return NULL; + + spin_lock_irqsave(&iommu_table_lock, flags); + + table = irq_lookup_table[devid]; + if (table) goto out_unlock; - /* Initialize table spin-lock */ - raw_spin_lock_init(&table->lock); - - table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_ATOMIC); - if (!table->table) { - kfree(table); - table = NULL; - goto out_unlock; + table = irq_lookup_table[alias]; + if (table) { + set_remap_table_entry(iommu, devid, table); + goto out_wait; } - if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) - memset(table->table, 0, - MAX_IRQS_PER_TABLE * sizeof(u32)); - else - memset(table->table, 0, - (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2))); - + table = new_table; + new_table = NULL; set_remap_table_entry(iommu, devid, table); if (devid != alias) set_remap_table_entry(iommu, alias, table); -out: +out_wait: iommu_completion_wait(iommu); out_unlock: spin_unlock_irqrestore(&iommu_table_lock, flags); + if (new_table) { + kmem_cache_free(amd_iommu_irq_cache, new_table->table); + kfree(new_table); + } return table; } From 2cd1083d79a0a8c223af430ca97884c28a1e2fc0 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:41 +0100 Subject: [PATCH 38/53] iommu/amd: Make amd_iommu_devtable_lock a spin_lock Before commit 0bb6e243d7fb ("iommu/amd: Support IOMMU_DOMAIN_DMA type allocation") amd_iommu_devtable_lock had a read_lock() user but now there are none. In fact, after the mentioned commit we had only write_lock() user of the lock. Since there is no reason to keep it as writer lock, change its type to a spin_lock. I *think* that we might even be able to remove the lock because all its current user seem to have their own protection. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index c493d345b3ef..23e26a4b708e 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -80,7 +80,7 @@ */ #define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38)) -static DEFINE_RWLOCK(amd_iommu_devtable_lock); +static DEFINE_SPINLOCK(amd_iommu_devtable_lock); static DEFINE_SPINLOCK(pd_bitmap_lock); static DEFINE_SPINLOCK(iommu_table_lock); @@ -2097,9 +2097,9 @@ static int attach_device(struct device *dev, } skip_ats_check: - write_lock_irqsave(&amd_iommu_devtable_lock, flags); + spin_lock_irqsave(&amd_iommu_devtable_lock, flags); ret = __attach_device(dev_data, domain); - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + spin_unlock_irqrestore(&amd_iommu_devtable_lock, flags); /* * We might boot into a crash-kernel here. The crashed kernel @@ -2149,9 +2149,9 @@ static void detach_device(struct device *dev) domain = dev_data->domain; /* lock device table */ - write_lock_irqsave(&amd_iommu_devtable_lock, flags); + spin_lock_irqsave(&amd_iommu_devtable_lock, flags); __detach_device(dev_data); - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + spin_unlock_irqrestore(&amd_iommu_devtable_lock, flags); if (!dev_is_pci(dev)) return; @@ -2814,7 +2814,7 @@ static void cleanup_domain(struct protection_domain *domain) struct iommu_dev_data *entry; unsigned long flags; - write_lock_irqsave(&amd_iommu_devtable_lock, flags); + spin_lock_irqsave(&amd_iommu_devtable_lock, flags); while (!list_empty(&domain->dev_list)) { entry = list_first_entry(&domain->dev_list, @@ -2822,7 +2822,7 @@ static void cleanup_domain(struct protection_domain *domain) __detach_device(entry); } - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + spin_unlock_irqrestore(&amd_iommu_devtable_lock, flags); } static void protection_domain_free(struct protection_domain *domain) From 29d049be9438278c47253a74cf8d0ddf36bd5d68 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 22 Mar 2018 16:22:42 +0100 Subject: [PATCH 39/53] iommu/amd: Return proper error code in irq_remapping_alloc() In the unlikely case when alloc_irq_table() is not able to return a remap table then "ret" will be assigned with an error code. Later, the code checks `index' and if it is negative (which it is because it is initialized with `-1') and then then function properly aborts but returns `-1' instead `-ENOMEM' what was intended. In order to correct this, I assign -ENOMEM to index. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 23e26a4b708e..6107e24bed8a 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -4124,7 +4124,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, struct amd_ir_data *data = NULL; struct irq_cfg *cfg; int i, ret, devid; - int index = -1; + int index; if (!info) return -EINVAL; @@ -4166,7 +4166,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, WARN_ON(table->min_index != 32); index = info->ioapic_pin; } else { - ret = -ENOMEM; + index = -ENOMEM; } } else { bool align = (info->type == X86_IRQ_ALLOC_TYPE_MSI); From 98b72b94def91be0d7918cbba6a150d97047f175 Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 23 Mar 2018 15:38:01 +0800 Subject: [PATCH 40/53] iommu/rockchip: Prohibit unbind and remove Removal of IOMMUs cannot be done reliably. This is similar to exynos iommu driver. Signed-off-by: Jeffy Chen Reviewed-by: Tomasz Figa Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 6a3719e118da..e7fb824d123e 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1197,18 +1197,6 @@ static int rk_iommu_probe(struct platform_device *pdev) return err; } -static int rk_iommu_remove(struct platform_device *pdev) -{ - struct rk_iommu *iommu = platform_get_drvdata(pdev); - - if (iommu) { - iommu_device_sysfs_remove(&iommu->iommu); - iommu_device_unregister(&iommu->iommu); - } - - return 0; -} - static void rk_iommu_shutdown(struct platform_device *pdev) { struct rk_iommu *iommu = platform_get_drvdata(pdev); @@ -1234,11 +1222,11 @@ MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids); static struct platform_driver rk_iommu_driver = { .probe = rk_iommu_probe, - .remove = rk_iommu_remove, .shutdown = rk_iommu_shutdown, .driver = { .name = "rk_iommu", .of_match_table = rk_iommu_dt_ids, + .suppress_bind_attrs = true, }, }; @@ -1266,14 +1254,7 @@ static int __init rk_iommu_init(void) platform_driver_unregister(&rk_iommu_domain_driver); return ret; } -static void __exit rk_iommu_exit(void) -{ - platform_driver_unregister(&rk_iommu_driver); - platform_driver_unregister(&rk_iommu_domain_driver); -} - subsys_initcall(rk_iommu_init); -module_exit(rk_iommu_exit); MODULE_DESCRIPTION("IOMMU API for Rockchip"); MODULE_AUTHOR("Simon Xue and Daniel Kurtz "); From 6d9ffaad7eddefaa0a166ba612665aef5264e352 Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 23 Mar 2018 15:38:02 +0800 Subject: [PATCH 41/53] iommu/rockchip: Fix error handling in probe Add missing iommu_device_sysfs_remove in error path. Signed-off-by: Jeffy Chen Reviewed-by: Tomasz Figa Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index e7fb824d123e..73117dbe839e 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1193,6 +1193,8 @@ static int rk_iommu_probe(struct platform_device *pdev) iommu_device_set_ops(&iommu->iommu, &rk_iommu_ops); err = iommu_device_register(&iommu->iommu); + if (err) + iommu_device_sysfs_remove(&iommu->iommu); return err; } From d0b912bd4c23fd8589584b1b0d582acb1230c795 Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 23 Mar 2018 15:38:03 +0800 Subject: [PATCH 42/53] iommu/rockchip: Request irqs in rk_iommu_probe() Move request_irq to the end of rk_iommu_probe(). Suggested-by: Robin Murphy Signed-off-by: Jeffy Chen Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 38 ++++++++-------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 73117dbe839e..ec3ff936aa60 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -90,8 +90,6 @@ struct rk_iommu { struct device *dev; void __iomem **bases; int num_mmu; - int *irq; - int num_irq; bool reset_disabled; struct iommu_device iommu; struct list_head node; /* entry in rk_iommu_domain.iommus */ @@ -830,13 +828,6 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, iommu->domain = domain; - for (i = 0; i < iommu->num_irq; i++) { - ret = devm_request_irq(iommu->dev, iommu->irq[i], rk_iommu_irq, - IRQF_SHARED, dev_name(dev), iommu); - if (ret) - return ret; - } - for (i = 0; i < iommu->num_mmu; i++) { rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, rk_domain->dt_dma); @@ -885,9 +876,6 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, } rk_iommu_disable_stall(iommu); - for (i = 0; i < iommu->num_irq; i++) - devm_free_irq(iommu->dev, iommu->irq[i], iommu); - iommu->domain = NULL; dev_dbg(dev, "Detached from iommu domain\n"); @@ -1138,7 +1126,7 @@ static int rk_iommu_probe(struct platform_device *pdev) struct rk_iommu *iommu; struct resource *res; int num_res = pdev->num_resources; - int err, i; + int err, i, irq; iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); if (!iommu) @@ -1165,23 +1153,15 @@ static int rk_iommu_probe(struct platform_device *pdev) if (iommu->num_mmu == 0) return PTR_ERR(iommu->bases[0]); - iommu->num_irq = platform_irq_count(pdev); - if (iommu->num_irq < 0) - return iommu->num_irq; - if (iommu->num_irq == 0) - return -ENXIO; + i = 0; + while ((irq = platform_get_irq(pdev, i++)) != -ENXIO) { + if (irq < 0) + return irq; - iommu->irq = devm_kcalloc(dev, iommu->num_irq, sizeof(*iommu->irq), - GFP_KERNEL); - if (!iommu->irq) - return -ENOMEM; - - for (i = 0; i < iommu->num_irq; i++) { - iommu->irq[i] = platform_get_irq(pdev, i); - if (iommu->irq[i] < 0) { - dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq[i]); - return -ENXIO; - } + err = devm_request_irq(iommu->dev, irq, rk_iommu_irq, + IRQF_SHARED, dev_name(dev), iommu); + if (err) + return err; } iommu->reset_disabled = device_property_read_bool(dev, From f6717d727c9a2b0742ac23234a347db0d3ad718a Mon Sep 17 00:00:00 2001 From: Tomasz Figa Date: Fri, 23 Mar 2018 15:38:04 +0800 Subject: [PATCH 43/53] iommu/rockchip: Fix error handling in attach Currently if the driver encounters an error while attaching device, it will leave the IOMMU in an inconsistent state. Even though it shouldn't really happen in reality, let's just add proper error path to keep things consistent. Signed-off-by: Tomasz Figa Signed-off-by: Jeffy Chen Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index ec3ff936aa60..5cbd5ceeb039 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -824,7 +824,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, ret = rk_iommu_force_reset(iommu); if (ret) - return ret; + goto out_disable_stall; iommu->domain = domain; @@ -837,7 +837,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, ret = rk_iommu_enable_paging(iommu); if (ret) - return ret; + goto out_disable_stall; spin_lock_irqsave(&rk_domain->iommus_lock, flags); list_add_tail(&iommu->node, &rk_domain->iommus); @@ -845,9 +845,9 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, dev_dbg(dev, "Attached to iommu domain\n"); +out_disable_stall: rk_iommu_disable_stall(iommu); - - return 0; + return ret; } static void rk_iommu_detach_device(struct iommu_domain *domain, From 0416bf6479acef646ad8c7396f73ecd936aa84bb Mon Sep 17 00:00:00 2001 From: Tomasz Figa Date: Fri, 23 Mar 2018 15:38:05 +0800 Subject: [PATCH 44/53] iommu/rockchip: Use iopoll helpers to wait for hardware This patch converts the rockchip-iommu driver to use the in-kernel iopoll helpers to wait for certain status bits to change in registers instead of an open-coded custom macro. Signed-off-by: Tomasz Figa Signed-off-by: Jeffy Chen Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 75 ++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 5cbd5ceeb039..7130d92ff79c 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -36,7 +36,10 @@ #define RK_MMU_AUTO_GATING 0x24 #define DTE_ADDR_DUMMY 0xCAFEBABE -#define FORCE_RESET_TIMEOUT 100 /* ms */ + +#define RK_MMU_POLL_PERIOD_US 100 +#define RK_MMU_FORCE_RESET_TIMEOUT_US 100000 +#define RK_MMU_POLL_TIMEOUT_US 1000 /* RK_MMU_STATUS fields */ #define RK_MMU_STATUS_PAGING_ENABLED BIT(0) @@ -73,8 +76,6 @@ */ #define RK_IOMMU_PGSIZE_BITMAP 0x007ff000 -#define IOMMU_REG_POLL_COUNT_FAST 1000 - struct rk_iommu_domain { struct list_head iommus; struct platform_device *pdev; @@ -109,27 +110,6 @@ static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) return container_of(dom, struct rk_iommu_domain, domain); } -/** - * Inspired by _wait_for in intel_drv.h - * This is NOT safe for use in interrupt context. - * - * Note that it's important that we check the condition again after having - * timed out, since the timeout could be due to preemption or similar and - * we've never had a chance to check the condition before the timeout. - */ -#define rk_wait_for(COND, MS) ({ \ - unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1; \ - int ret__ = 0; \ - while (!(COND)) { \ - if (time_after(jiffies, timeout__)) { \ - ret__ = (COND) ? 0 : -ETIMEDOUT; \ - break; \ - } \ - usleep_range(50, 100); \ - } \ - ret__; \ -}) - /* * The Rockchip rk3288 iommu uses a 2-level page table. * The first level is the "Directory Table" (DT). @@ -333,9 +313,21 @@ static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) return enable; } +static bool rk_iommu_is_reset_done(struct rk_iommu *iommu) +{ + bool done = true; + int i; + + for (i = 0; i < iommu->num_mmu; i++) + done &= rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR) == 0; + + return done; +} + static int rk_iommu_enable_stall(struct rk_iommu *iommu) { int ret, i; + bool val; if (rk_iommu_is_stall_active(iommu)) return 0; @@ -346,7 +338,9 @@ static int rk_iommu_enable_stall(struct rk_iommu *iommu) rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL); - ret = rk_wait_for(rk_iommu_is_stall_active(iommu), 1); + ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val, + val, RK_MMU_POLL_PERIOD_US, + RK_MMU_POLL_TIMEOUT_US); if (ret) for (i = 0; i < iommu->num_mmu; i++) dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n", @@ -358,13 +352,16 @@ static int rk_iommu_enable_stall(struct rk_iommu *iommu) static int rk_iommu_disable_stall(struct rk_iommu *iommu) { int ret, i; + bool val; if (!rk_iommu_is_stall_active(iommu)) return 0; rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL); - ret = rk_wait_for(!rk_iommu_is_stall_active(iommu), 1); + ret = readx_poll_timeout(rk_iommu_is_stall_active, iommu, val, + !val, RK_MMU_POLL_PERIOD_US, + RK_MMU_POLL_TIMEOUT_US); if (ret) for (i = 0; i < iommu->num_mmu; i++) dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n", @@ -376,13 +373,16 @@ static int rk_iommu_disable_stall(struct rk_iommu *iommu) static int rk_iommu_enable_paging(struct rk_iommu *iommu) { int ret, i; + bool val; if (rk_iommu_is_paging_enabled(iommu)) return 0; rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING); - ret = rk_wait_for(rk_iommu_is_paging_enabled(iommu), 1); + ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val, + val, RK_MMU_POLL_PERIOD_US, + RK_MMU_POLL_TIMEOUT_US); if (ret) for (i = 0; i < iommu->num_mmu; i++) dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n", @@ -394,13 +394,16 @@ static int rk_iommu_enable_paging(struct rk_iommu *iommu) static int rk_iommu_disable_paging(struct rk_iommu *iommu) { int ret, i; + bool val; if (!rk_iommu_is_paging_enabled(iommu)) return 0; rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING); - ret = rk_wait_for(!rk_iommu_is_paging_enabled(iommu), 1); + ret = readx_poll_timeout(rk_iommu_is_paging_enabled, iommu, val, + !val, RK_MMU_POLL_PERIOD_US, + RK_MMU_POLL_TIMEOUT_US); if (ret) for (i = 0; i < iommu->num_mmu; i++) dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n", @@ -413,6 +416,7 @@ static int rk_iommu_force_reset(struct rk_iommu *iommu) { int ret, i; u32 dte_addr; + bool val; if (iommu->reset_disabled) return 0; @@ -433,13 +437,12 @@ static int rk_iommu_force_reset(struct rk_iommu *iommu) rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET); - for (i = 0; i < iommu->num_mmu; i++) { - ret = rk_wait_for(rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR) == 0x00000000, - FORCE_RESET_TIMEOUT); - if (ret) { - dev_err(iommu->dev, "FORCE_RESET command timed out\n"); - return ret; - } + ret = readx_poll_timeout(rk_iommu_is_reset_done, iommu, val, + val, RK_MMU_FORCE_RESET_TIMEOUT_US, + RK_MMU_POLL_TIMEOUT_US); + if (ret) { + dev_err(iommu->dev, "FORCE_RESET command timed out\n"); + return ret; } return 0; From bf2a5e717a47d042ca5f9f438cf186633f23033b Mon Sep 17 00:00:00 2001 From: Tomasz Figa Date: Fri, 23 Mar 2018 15:38:06 +0800 Subject: [PATCH 45/53] iommu/rockchip: Fix TLB flush of secondary IOMMUs Due to the bug in current code, only first IOMMU has the TLB lines flushed in rk_iommu_zap_lines. This patch fixes the inner loop to execute for all IOMMUs and properly flush the TLB. Signed-off-by: Tomasz Figa Signed-off-by: Jeffy Chen Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 7130d92ff79c..c3df53e480ea 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -274,19 +274,21 @@ static void rk_iommu_base_command(void __iomem *base, u32 command) { writel(command, base + RK_MMU_COMMAND); } -static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova, +static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova_start, size_t size) { int i; - - dma_addr_t iova_end = iova + size; + dma_addr_t iova_end = iova_start + size; /* * TODO(djkurtz): Figure out when it is more efficient to shootdown the * entire iotlb rather than iterate over individual iovas. */ - for (i = 0; i < iommu->num_mmu; i++) - for (; iova < iova_end; iova += SPAGE_SIZE) + for (i = 0; i < iommu->num_mmu; i++) { + dma_addr_t iova; + + for (iova = iova_start; iova < iova_end; iova += SPAGE_SIZE) rk_iommu_write(iommu->bases[i], RK_MMU_ZAP_ONE_LINE, iova); + } } static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) From f2e3a5f557ad27f6a6f447717090a39cea238d6a Mon Sep 17 00:00:00 2001 From: Tomasz Figa Date: Fri, 23 Mar 2018 15:38:08 +0800 Subject: [PATCH 46/53] iommu/rockchip: Control clocks needed to access the IOMMU Current code relies on master driver enabling necessary clocks before IOMMU is accessed, however there are cases when the IOMMU should be accessed while the master is not running yet, for example allocating V4L2 videobuf2 buffers, which is done by the VB2 framework using DMA mapping API and doesn't engage the master driver at all. This patch fixes the problem by letting clocks needed for IOMMU operation to be listed in Device Tree and making the driver enable them for the time of accessing the hardware. Signed-off-by: Tomasz Figa Signed-off-by: Jeffy Chen Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 50 ++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index c3df53e480ea..50ab9fd4eeb0 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -4,6 +4,7 @@ * published by the Free Software Foundation. */ +#include #include #include #include @@ -87,10 +88,17 @@ struct rk_iommu_domain { struct iommu_domain domain; }; +/* list of clocks required by IOMMU */ +static const char * const rk_iommu_clocks[] = { + "aclk", "iface", +}; + struct rk_iommu { struct device *dev; void __iomem **bases; int num_mmu; + struct clk_bulk_data *clocks; + int num_clocks; bool reset_disabled; struct iommu_device iommu; struct list_head node; /* entry in rk_iommu_domain.iommus */ @@ -506,6 +514,8 @@ static irqreturn_t rk_iommu_irq(int irq, void *dev_id) irqreturn_t ret = IRQ_NONE; int i; + WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); + for (i = 0; i < iommu->num_mmu; i++) { int_status = rk_iommu_read(iommu->bases[i], RK_MMU_INT_STATUS); if (int_status == 0) @@ -552,6 +562,8 @@ static irqreturn_t rk_iommu_irq(int irq, void *dev_id) rk_iommu_write(iommu->bases[i], RK_MMU_INT_CLEAR, int_status); } + clk_bulk_disable(iommu->num_clocks, iommu->clocks); + return ret; } @@ -594,7 +606,9 @@ static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, list_for_each(pos, &rk_domain->iommus) { struct rk_iommu *iommu; iommu = list_entry(pos, struct rk_iommu, node); + WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); rk_iommu_zap_lines(iommu, iova, size); + clk_bulk_disable(iommu->num_clocks, iommu->clocks); } spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); } @@ -823,10 +837,14 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, if (!iommu) return 0; - ret = rk_iommu_enable_stall(iommu); + ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks); if (ret) return ret; + ret = rk_iommu_enable_stall(iommu); + if (ret) + goto out_disable_clocks; + ret = rk_iommu_force_reset(iommu); if (ret) goto out_disable_stall; @@ -852,6 +870,8 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, out_disable_stall: rk_iommu_disable_stall(iommu); +out_disable_clocks: + clk_bulk_disable(iommu->num_clocks, iommu->clocks); return ret; } @@ -873,6 +893,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); /* Ignore error while disabling, just keep going */ + WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); rk_iommu_enable_stall(iommu); rk_iommu_disable_paging(iommu); for (i = 0; i < iommu->num_mmu; i++) { @@ -880,6 +901,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0); } rk_iommu_disable_stall(iommu); + clk_bulk_disable(iommu->num_clocks, iommu->clocks); iommu->domain = NULL; @@ -1172,15 +1194,37 @@ static int rk_iommu_probe(struct platform_device *pdev) iommu->reset_disabled = device_property_read_bool(dev, "rockchip,disable-mmu-reset"); - err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev)); + iommu->num_clocks = ARRAY_SIZE(rk_iommu_clocks); + iommu->clocks = devm_kcalloc(iommu->dev, iommu->num_clocks, + sizeof(*iommu->clocks), GFP_KERNEL); + if (!iommu->clocks) + return -ENOMEM; + + for (i = 0; i < iommu->num_clocks; ++i) + iommu->clocks[i].id = rk_iommu_clocks[i]; + + err = devm_clk_bulk_get(iommu->dev, iommu->num_clocks, iommu->clocks); if (err) return err; + err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks); + if (err) + return err; + + err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev)); + if (err) + goto err_unprepare_clocks; + iommu_device_set_ops(&iommu->iommu, &rk_iommu_ops); err = iommu_device_register(&iommu->iommu); if (err) - iommu_device_sysfs_remove(&iommu->iommu); + goto err_remove_sysfs; + return 0; +err_remove_sysfs: + iommu_device_sysfs_remove(&iommu->iommu); +err_unprepare_clocks: + clk_bulk_unprepare(iommu->num_clocks, iommu->clocks); return err; } From 8fa9eb39c614acbabaecefd5507a426b45fa4704 Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 23 Mar 2018 15:38:09 +0800 Subject: [PATCH 47/53] dt-bindings: iommu/rockchip: Add clock property Add clock property, since we are going to control clocks in rockchip iommu driver. Signed-off-by: Jeffy Chen Reviewed-by: Robin Murphy Reviewed-by: Rob Herring Signed-off-by: Joerg Roedel --- Documentation/devicetree/bindings/iommu/rockchip,iommu.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt b/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt index 2098f7732264..6ecefea1c6f9 100644 --- a/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt +++ b/Documentation/devicetree/bindings/iommu/rockchip,iommu.txt @@ -14,6 +14,11 @@ Required properties: "single-master" device, and needs no additional information to associate with its master device. See: Documentation/devicetree/bindings/iommu/iommu.txt +- clocks : A list of clocks required for the IOMMU to be accessible by + the host CPU. +- clock-names : Should contain the following: + "iface" - Main peripheral bus clock (PCLK/HCL) (required) + "aclk" - AXI bus clock (required) Optional properties: - rockchip,disable-mmu-reset : Don't use the mmu reset operation. @@ -27,5 +32,7 @@ Example: reg = <0xff940300 0x100>; interrupts = ; interrupt-names = "vopl_mmu"; + clocks = <&cru ACLK_VOP1>, <&cru HCLK_VOP1>; + clock-names = "aclk", "iface"; #iommu-cells = <0>; }; From 9176a303d971dc0fb35469c531c0d263667d2277 Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 23 Mar 2018 15:38:10 +0800 Subject: [PATCH 48/53] iommu/rockchip: Use IOMMU device for dma mapping operations Use the first registered IOMMU device for dma mapping operations, and drop the domain platform device. This is similar to exynos iommu driver. Signed-off-by: Jeffy Chen Reviewed-by: Tomasz Figa Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 85 ++++++++++------------------------ 1 file changed, 24 insertions(+), 61 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 50ab9fd4eeb0..7970d21b9858 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -79,7 +79,6 @@ struct rk_iommu_domain { struct list_head iommus; - struct platform_device *pdev; u32 *dt; /* page directory table */ dma_addr_t dt_dma; spinlock_t iommus_lock; /* lock for iommus list */ @@ -105,12 +104,14 @@ struct rk_iommu { struct iommu_domain *domain; /* domain to which iommu is attached */ }; +static struct device *dma_dev; + static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, unsigned int count) { size_t size = count * sizeof(u32); /* count of u32 entry */ - dma_sync_single_for_device(&dom->pdev->dev, dma, size, DMA_TO_DEVICE); + dma_sync_single_for_device(dma_dev, dma, size, DMA_TO_DEVICE); } static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) @@ -625,7 +626,6 @@ static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, dma_addr_t iova) { - struct device *dev = &rk_domain->pdev->dev; u32 *page_table, *dte_addr; u32 dte_index, dte; phys_addr_t pt_phys; @@ -643,9 +643,9 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, if (!page_table) return ERR_PTR(-ENOMEM); - pt_dma = dma_map_single(dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); - if (dma_mapping_error(dev, pt_dma)) { - dev_err(dev, "DMA mapping error while allocating page table\n"); + pt_dma = dma_map_single(dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev, pt_dma)) { + dev_err(dma_dev, "DMA mapping error while allocating page table\n"); free_page((unsigned long)page_table); return ERR_PTR(-ENOMEM); } @@ -911,29 +911,20 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) { struct rk_iommu_domain *rk_domain; - struct platform_device *pdev; - struct device *iommu_dev; if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) return NULL; - /* Register a pdev per domain, so DMA API can base on this *dev - * even some virtual master doesn't have an iommu slave - */ - pdev = platform_device_register_simple("rk_iommu_domain", - PLATFORM_DEVID_AUTO, NULL, 0); - if (IS_ERR(pdev)) + if (!dma_dev) return NULL; - rk_domain = devm_kzalloc(&pdev->dev, sizeof(*rk_domain), GFP_KERNEL); + rk_domain = devm_kzalloc(dma_dev, sizeof(*rk_domain), GFP_KERNEL); if (!rk_domain) - goto err_unreg_pdev; - - rk_domain->pdev = pdev; + return NULL; if (type == IOMMU_DOMAIN_DMA && iommu_get_dma_cookie(&rk_domain->domain)) - goto err_unreg_pdev; + return NULL; /* * rk32xx iommus use a 2 level pagetable. @@ -944,11 +935,10 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) if (!rk_domain->dt) goto err_put_cookie; - iommu_dev = &pdev->dev; - rk_domain->dt_dma = dma_map_single(iommu_dev, rk_domain->dt, + rk_domain->dt_dma = dma_map_single(dma_dev, rk_domain->dt, SPAGE_SIZE, DMA_TO_DEVICE); - if (dma_mapping_error(iommu_dev, rk_domain->dt_dma)) { - dev_err(iommu_dev, "DMA map error for DT\n"); + if (dma_mapping_error(dma_dev, rk_domain->dt_dma)) { + dev_err(dma_dev, "DMA map error for DT\n"); goto err_free_dt; } @@ -969,8 +959,6 @@ err_free_dt: err_put_cookie: if (type == IOMMU_DOMAIN_DMA) iommu_put_dma_cookie(&rk_domain->domain); -err_unreg_pdev: - platform_device_unregister(pdev); return NULL; } @@ -987,20 +975,18 @@ static void rk_iommu_domain_free(struct iommu_domain *domain) if (rk_dte_is_pt_valid(dte)) { phys_addr_t pt_phys = rk_dte_pt_address(dte); u32 *page_table = phys_to_virt(pt_phys); - dma_unmap_single(&rk_domain->pdev->dev, pt_phys, + dma_unmap_single(dma_dev, pt_phys, SPAGE_SIZE, DMA_TO_DEVICE); free_page((unsigned long)page_table); } } - dma_unmap_single(&rk_domain->pdev->dev, rk_domain->dt_dma, + dma_unmap_single(dma_dev, rk_domain->dt_dma, SPAGE_SIZE, DMA_TO_DEVICE); free_page((unsigned long)rk_domain->dt); if (domain->type == IOMMU_DOMAIN_DMA) iommu_put_dma_cookie(&rk_domain->domain); - - platform_device_unregister(rk_domain->pdev); } static bool rk_iommu_is_dev_iommu_master(struct device *dev) @@ -1123,30 +1109,6 @@ static const struct iommu_ops rk_iommu_ops = { .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, }; -static int rk_iommu_domain_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - - dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL); - if (!dev->dma_parms) - return -ENOMEM; - - /* Set dma_ops for dev, otherwise it would be dummy_dma_ops */ - arch_setup_dma_ops(dev, 0, DMA_BIT_MASK(32), NULL, false); - - dma_set_max_seg_size(dev, DMA_BIT_MASK(32)); - dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32)); - - return 0; -} - -static struct platform_driver rk_iommu_domain_driver = { - .probe = rk_iommu_domain_probe, - .driver = { - .name = "rk_iommu_domain", - }, -}; - static int rk_iommu_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1220,6 +1182,14 @@ static int rk_iommu_probe(struct platform_device *pdev) if (err) goto err_remove_sysfs; + /* + * Use the first registered IOMMU device for domain to use with DMA + * API, since a domain might not physically correspond to a single + * IOMMU device.. + */ + if (!dma_dev) + dma_dev = &pdev->dev; + return 0; err_remove_sysfs: iommu_device_sysfs_remove(&iommu->iommu); @@ -1276,14 +1246,7 @@ static int __init rk_iommu_init(void) if (ret) return ret; - ret = platform_driver_register(&rk_iommu_domain_driver); - if (ret) - return ret; - - ret = platform_driver_register(&rk_iommu_driver); - if (ret) - platform_driver_unregister(&rk_iommu_domain_driver); - return ret; + return platform_driver_register(&rk_iommu_driver); } subsys_initcall(rk_iommu_init); From 5fd577c3eac3bdb9aebfec01e9b3d7f07a14f2dd Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 23 Mar 2018 15:38:11 +0800 Subject: [PATCH 49/53] iommu/rockchip: Use OF_IOMMU to attach devices automatically Converts the rockchip-iommu driver to use the OF_IOMMU infrastructure, which allows attaching master devices to their IOMMUs automatically according to DT properties. Signed-off-by: Jeffy Chen Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 137 ++++++++++----------------------- 1 file changed, 41 insertions(+), 96 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 7970d21b9858..bd8580b897e9 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -104,6 +105,10 @@ struct rk_iommu { struct iommu_domain *domain; /* domain to which iommu is attached */ }; +struct rk_iommudata { + struct rk_iommu *iommu; +}; + static struct device *dma_dev; static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, @@ -807,18 +812,9 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, static struct rk_iommu *rk_iommu_from_dev(struct device *dev) { - struct iommu_group *group; - struct device *iommu_dev; - struct rk_iommu *rk_iommu; + struct rk_iommudata *data = dev->archdata.iommu; - group = iommu_group_get(dev); - if (!group) - return NULL; - iommu_dev = iommu_group_get_iommudata(group); - rk_iommu = dev_get_drvdata(iommu_dev); - iommu_group_put(group); - - return rk_iommu; + return data ? data->iommu : NULL; } static int rk_iommu_attach_device(struct iommu_domain *domain, @@ -989,112 +985,55 @@ static void rk_iommu_domain_free(struct iommu_domain *domain) iommu_put_dma_cookie(&rk_domain->domain); } -static bool rk_iommu_is_dev_iommu_master(struct device *dev) -{ - struct device_node *np = dev->of_node; - int ret; - - /* - * An iommu master has an iommus property containing a list of phandles - * to iommu nodes, each with an #iommu-cells property with value 0. - */ - ret = of_count_phandle_with_args(np, "iommus", "#iommu-cells"); - return (ret > 0); -} - -static int rk_iommu_group_set_iommudata(struct iommu_group *group, - struct device *dev) -{ - struct device_node *np = dev->of_node; - struct platform_device *pd; - int ret; - struct of_phandle_args args; - - /* - * An iommu master has an iommus property containing a list of phandles - * to iommu nodes, each with an #iommu-cells property with value 0. - */ - ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0, - &args); - if (ret) { - dev_err(dev, "of_parse_phandle_with_args(%pOF) => %d\n", - np, ret); - return ret; - } - if (args.args_count != 0) { - dev_err(dev, "incorrect number of iommu params found for %pOF (found %d, expected 0)\n", - args.np, args.args_count); - return -EINVAL; - } - - pd = of_find_device_by_node(args.np); - of_node_put(args.np); - if (!pd) { - dev_err(dev, "iommu %pOF not found\n", args.np); - return -EPROBE_DEFER; - } - - /* TODO(djkurtz): handle multiple slave iommus for a single master */ - iommu_group_set_iommudata(group, &pd->dev, NULL); - - return 0; -} - static int rk_iommu_add_device(struct device *dev) { struct iommu_group *group; struct rk_iommu *iommu; - int ret; - - if (!rk_iommu_is_dev_iommu_master(dev)) - return -ENODEV; - - group = iommu_group_get(dev); - if (!group) { - group = iommu_group_alloc(); - if (IS_ERR(group)) { - dev_err(dev, "Failed to allocate IOMMU group\n"); - return PTR_ERR(group); - } - } - - ret = iommu_group_add_device(group, dev); - if (ret) - goto err_put_group; - - ret = rk_iommu_group_set_iommudata(group, dev); - if (ret) - goto err_remove_device; iommu = rk_iommu_from_dev(dev); - if (iommu) - iommu_device_link(&iommu->iommu, dev); + if (!iommu) + return -ENODEV; + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return PTR_ERR(group); iommu_group_put(group); + iommu_device_link(&iommu->iommu, dev); + return 0; - -err_remove_device: - iommu_group_remove_device(dev); -err_put_group: - iommu_group_put(group); - return ret; } static void rk_iommu_remove_device(struct device *dev) { struct rk_iommu *iommu; - if (!rk_iommu_is_dev_iommu_master(dev)) - return; - iommu = rk_iommu_from_dev(dev); - if (iommu) - iommu_device_unlink(&iommu->iommu, dev); + iommu_device_unlink(&iommu->iommu, dev); iommu_group_remove_device(dev); } +static int rk_iommu_of_xlate(struct device *dev, + struct of_phandle_args *args) +{ + struct platform_device *iommu_dev; + struct rk_iommudata *data; + + data = devm_kzalloc(dma_dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + iommu_dev = of_find_device_by_node(args->np); + + data->iommu = platform_get_drvdata(iommu_dev); + dev->archdata.iommu = data; + + of_dev_put(iommu_dev); + + return 0; +} + static const struct iommu_ops rk_iommu_ops = { .domain_alloc = rk_iommu_domain_alloc, .domain_free = rk_iommu_domain_free, @@ -1106,7 +1045,9 @@ static const struct iommu_ops rk_iommu_ops = { .add_device = rk_iommu_add_device, .remove_device = rk_iommu_remove_device, .iova_to_phys = rk_iommu_iova_to_phys, + .device_group = generic_device_group, .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, + .of_xlate = rk_iommu_of_xlate, }; static int rk_iommu_probe(struct platform_device *pdev) @@ -1178,6 +1119,8 @@ static int rk_iommu_probe(struct platform_device *pdev) goto err_unprepare_clocks; iommu_device_set_ops(&iommu->iommu, &rk_iommu_ops); + iommu_device_set_fwnode(&iommu->iommu, &dev->of_node->fwnode); + err = iommu_device_register(&iommu->iommu); if (err) goto err_remove_sysfs; @@ -1250,6 +1193,8 @@ static int __init rk_iommu_init(void) } subsys_initcall(rk_iommu_init); +IOMMU_OF_DECLARE(rk_iommu_of, "rockchip,iommu"); + MODULE_DESCRIPTION("IOMMU API for Rockchip"); MODULE_AUTHOR("Simon Xue and Daniel Kurtz "); MODULE_ALIAS("platform:rockchip-iommu"); From 4d88a8a4c345cd16f634df855148cfb2a59a204a Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 23 Mar 2018 15:38:12 +0800 Subject: [PATCH 50/53] iommu/rockchip: Fix error handling in init It's hard to undo bus_set_iommu() in the error path, so move it to the end of rk_iommu_probe(). Signed-off-by: Jeffy Chen Reviewed-by: Tomasz Figa Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index bd8580b897e9..0ce5e8a0658c 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1133,6 +1133,8 @@ static int rk_iommu_probe(struct platform_device *pdev) if (!dma_dev) dma_dev = &pdev->dev; + bus_set_iommu(&platform_bus_type, &rk_iommu_ops); + return 0; err_remove_sysfs: iommu_device_sysfs_remove(&iommu->iommu); @@ -1176,19 +1178,6 @@ static struct platform_driver rk_iommu_driver = { static int __init rk_iommu_init(void) { - struct device_node *np; - int ret; - - np = of_find_matching_node(NULL, rk_iommu_dt_ids); - if (!np) - return 0; - - of_node_put(np); - - ret = bus_set_iommu(&platform_bus_type, &rk_iommu_ops); - if (ret) - return ret; - return platform_driver_register(&rk_iommu_driver); } subsys_initcall(rk_iommu_init); From 0f181d3cf7d984b1af33d34557a8285852b0d3ec Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 23 Mar 2018 15:38:13 +0800 Subject: [PATCH 51/53] iommu/rockchip: Add runtime PM support When the power domain is powered off, the IOMMU cannot be accessed and register programming must be deferred until the power domain becomes enabled. Add runtime PM support, and use runtime PM device link from IOMMU to master to enable and disable IOMMU. Signed-off-by: Jeffy Chen Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 185 +++++++++++++++++++++++---------- 1 file changed, 131 insertions(+), 54 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 0ce5e8a0658c..9f6f74689464 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -106,6 +107,7 @@ struct rk_iommu { }; struct rk_iommudata { + struct device_link *link; /* runtime PM link from IOMMU to master */ struct rk_iommu *iommu; }; @@ -520,7 +522,11 @@ static irqreturn_t rk_iommu_irq(int irq, void *dev_id) irqreturn_t ret = IRQ_NONE; int i; - WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); + if (WARN_ON(!pm_runtime_get_if_in_use(iommu->dev))) + return 0; + + if (WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks))) + goto out; for (i = 0; i < iommu->num_mmu; i++) { int_status = rk_iommu_read(iommu->bases[i], RK_MMU_INT_STATUS); @@ -570,6 +576,8 @@ static irqreturn_t rk_iommu_irq(int irq, void *dev_id) clk_bulk_disable(iommu->num_clocks, iommu->clocks); +out: + pm_runtime_put(iommu->dev); return ret; } @@ -611,10 +619,17 @@ static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, spin_lock_irqsave(&rk_domain->iommus_lock, flags); list_for_each(pos, &rk_domain->iommus) { struct rk_iommu *iommu; + iommu = list_entry(pos, struct rk_iommu, node); - WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); - rk_iommu_zap_lines(iommu, iova, size); - clk_bulk_disable(iommu->num_clocks, iommu->clocks); + + /* Only zap TLBs of IOMMUs that are powered on. */ + if (pm_runtime_get_if_in_use(iommu->dev)) { + WARN_ON(clk_bulk_enable(iommu->num_clocks, + iommu->clocks)); + rk_iommu_zap_lines(iommu, iova, size); + clk_bulk_disable(iommu->num_clocks, iommu->clocks); + pm_runtime_put(iommu->dev); + } } spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); } @@ -817,21 +832,29 @@ static struct rk_iommu *rk_iommu_from_dev(struct device *dev) return data ? data->iommu : NULL; } -static int rk_iommu_attach_device(struct iommu_domain *domain, - struct device *dev) +/* Must be called with iommu powered on and attached */ +static void rk_iommu_disable(struct rk_iommu *iommu) { - struct rk_iommu *iommu; - struct rk_iommu_domain *rk_domain = to_rk_domain(domain); - unsigned long flags; - int ret, i; + int i; - /* - * Allow 'virtual devices' (e.g., drm) to attach to domain. - * Such a device does not belong to an iommu group. - */ - iommu = rk_iommu_from_dev(dev); - if (!iommu) - return 0; + /* Ignore error while disabling, just keep going */ + WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); + rk_iommu_enable_stall(iommu); + rk_iommu_disable_paging(iommu); + for (i = 0; i < iommu->num_mmu; i++) { + rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, 0); + rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0); + } + rk_iommu_disable_stall(iommu); + clk_bulk_disable(iommu->num_clocks, iommu->clocks); +} + +/* Must be called with iommu powered on and attached */ +static int rk_iommu_enable(struct rk_iommu *iommu) +{ + struct iommu_domain *domain = iommu->domain; + struct rk_iommu_domain *rk_domain = to_rk_domain(domain); + int ret, i; ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks); if (ret) @@ -845,8 +868,6 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, if (ret) goto out_disable_stall; - iommu->domain = domain; - for (i = 0; i < iommu->num_mmu; i++) { rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, rk_domain->dt_dma); @@ -855,14 +876,6 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, } ret = rk_iommu_enable_paging(iommu); - if (ret) - goto out_disable_stall; - - spin_lock_irqsave(&rk_domain->iommus_lock, flags); - list_add_tail(&iommu->node, &rk_domain->iommus); - spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); - - dev_dbg(dev, "Attached to iommu domain\n"); out_disable_stall: rk_iommu_disable_stall(iommu); @@ -877,31 +890,71 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, struct rk_iommu *iommu; struct rk_iommu_domain *rk_domain = to_rk_domain(domain); unsigned long flags; - int i; /* Allow 'virtual devices' (eg drm) to detach from domain */ iommu = rk_iommu_from_dev(dev); if (!iommu) return; + dev_dbg(dev, "Detaching from iommu domain\n"); + + /* iommu already detached */ + if (iommu->domain != domain) + return; + + iommu->domain = NULL; + spin_lock_irqsave(&rk_domain->iommus_lock, flags); list_del_init(&iommu->node); spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); - /* Ignore error while disabling, just keep going */ - WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); - rk_iommu_enable_stall(iommu); - rk_iommu_disable_paging(iommu); - for (i = 0; i < iommu->num_mmu; i++) { - rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, 0); - rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0); + if (pm_runtime_get_if_in_use(iommu->dev)) { + rk_iommu_disable(iommu); + pm_runtime_put(iommu->dev); } - rk_iommu_disable_stall(iommu); - clk_bulk_disable(iommu->num_clocks, iommu->clocks); +} - iommu->domain = NULL; +static int rk_iommu_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct rk_iommu *iommu; + struct rk_iommu_domain *rk_domain = to_rk_domain(domain); + unsigned long flags; + int ret; - dev_dbg(dev, "Detached from iommu domain\n"); + /* + * Allow 'virtual devices' (e.g., drm) to attach to domain. + * Such a device does not belong to an iommu group. + */ + iommu = rk_iommu_from_dev(dev); + if (!iommu) + return 0; + + dev_dbg(dev, "Attaching to iommu domain\n"); + + /* iommu already attached */ + if (iommu->domain == domain) + return 0; + + if (iommu->domain) + rk_iommu_detach_device(iommu->domain, dev); + + iommu->domain = domain; + + spin_lock_irqsave(&rk_domain->iommus_lock, flags); + list_add_tail(&iommu->node, &rk_domain->iommus); + spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); + + if (!pm_runtime_get_if_in_use(iommu->dev)) + return 0; + + ret = rk_iommu_enable(iommu); + if (ret) + rk_iommu_detach_device(iommu->domain, dev); + + pm_runtime_put(iommu->dev); + + return ret; } static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) @@ -989,10 +1042,13 @@ static int rk_iommu_add_device(struct device *dev) { struct iommu_group *group; struct rk_iommu *iommu; + struct rk_iommudata *data; + + data = dev->archdata.iommu; + if (!data) + return -ENODEV; iommu = rk_iommu_from_dev(dev); - if (!iommu) - return -ENODEV; group = iommu_group_get_for_dev(dev); if (IS_ERR(group)) @@ -1000,6 +1056,7 @@ static int rk_iommu_add_device(struct device *dev) iommu_group_put(group); iommu_device_link(&iommu->iommu, dev); + data->link = device_link_add(dev, iommu->dev, DL_FLAG_PM_RUNTIME); return 0; } @@ -1007,9 +1064,11 @@ static int rk_iommu_add_device(struct device *dev) static void rk_iommu_remove_device(struct device *dev) { struct rk_iommu *iommu; + struct rk_iommudata *data = dev->archdata.iommu; iommu = rk_iommu_from_dev(dev); + device_link_del(data->link); iommu_device_unlink(&iommu->iommu, dev); iommu_group_remove_device(dev); } @@ -1135,6 +1194,8 @@ static int rk_iommu_probe(struct platform_device *pdev) bus_set_iommu(&platform_bus_type, &rk_iommu_ops); + pm_runtime_enable(dev); + return 0; err_remove_sysfs: iommu_device_sysfs_remove(&iommu->iommu); @@ -1145,21 +1206,36 @@ err_unprepare_clocks: static void rk_iommu_shutdown(struct platform_device *pdev) { - struct rk_iommu *iommu = platform_get_drvdata(pdev); - - /* - * Be careful not to try to shutdown an otherwise unused - * IOMMU, as it is likely not to be clocked, and accessing it - * would just block. An IOMMU without a domain is likely to be - * unused, so let's use this as a (weak) guard. - */ - if (iommu && iommu->domain) { - rk_iommu_enable_stall(iommu); - rk_iommu_disable_paging(iommu); - rk_iommu_force_reset(iommu); - } + pm_runtime_force_suspend(&pdev->dev); } +static int __maybe_unused rk_iommu_suspend(struct device *dev) +{ + struct rk_iommu *iommu = dev_get_drvdata(dev); + + if (!iommu->domain) + return 0; + + rk_iommu_disable(iommu); + return 0; +} + +static int __maybe_unused rk_iommu_resume(struct device *dev) +{ + struct rk_iommu *iommu = dev_get_drvdata(dev); + + if (!iommu->domain) + return 0; + + return rk_iommu_enable(iommu); +} + +static const struct dev_pm_ops rk_iommu_pm_ops = { + SET_RUNTIME_PM_OPS(rk_iommu_suspend, rk_iommu_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) +}; + static const struct of_device_id rk_iommu_dt_ids[] = { { .compatible = "rockchip,iommu" }, { /* sentinel */ } @@ -1172,6 +1248,7 @@ static struct platform_driver rk_iommu_driver = { .driver = { .name = "rk_iommu", .of_match_table = rk_iommu_dt_ids, + .pm = &rk_iommu_pm_ops, .suppress_bind_attrs = true, }, }; From 57c26957bd87a9f3b84043d0ee9cf47852a82ff9 Mon Sep 17 00:00:00 2001 From: Jeffy Chen Date: Fri, 23 Mar 2018 15:38:14 +0800 Subject: [PATCH 52/53] iommu/rockchip: Support sharing IOMMU between masters There would be some masters sharing the same IOMMU device. Put them in the same iommu group and share the same iommu domain. Signed-off-by: Jeffy Chen Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 9f6f74689464..5fc8656c60f9 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -104,6 +104,7 @@ struct rk_iommu { struct iommu_device iommu; struct list_head node; /* entry in rk_iommu_domain.iommus */ struct iommu_domain *domain; /* domain to which iommu is attached */ + struct iommu_group *group; }; struct rk_iommudata { @@ -1073,6 +1074,15 @@ static void rk_iommu_remove_device(struct device *dev) iommu_group_remove_device(dev); } +static struct iommu_group *rk_iommu_device_group(struct device *dev) +{ + struct rk_iommu *iommu; + + iommu = rk_iommu_from_dev(dev); + + return iommu_group_ref_get(iommu->group); +} + static int rk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) { @@ -1104,7 +1114,7 @@ static const struct iommu_ops rk_iommu_ops = { .add_device = rk_iommu_add_device, .remove_device = rk_iommu_remove_device, .iova_to_phys = rk_iommu_iova_to_phys, - .device_group = generic_device_group, + .device_group = rk_iommu_device_group, .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, .of_xlate = rk_iommu_of_xlate, }; @@ -1173,9 +1183,15 @@ static int rk_iommu_probe(struct platform_device *pdev) if (err) return err; + iommu->group = iommu_group_alloc(); + if (IS_ERR(iommu->group)) { + err = PTR_ERR(iommu->group); + goto err_unprepare_clocks; + } + err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev)); if (err) - goto err_unprepare_clocks; + goto err_put_group; iommu_device_set_ops(&iommu->iommu, &rk_iommu_ops); iommu_device_set_fwnode(&iommu->iommu, &dev->of_node->fwnode); @@ -1199,6 +1215,8 @@ static int rk_iommu_probe(struct platform_device *pdev) return 0; err_remove_sysfs: iommu_device_sysfs_remove(&iommu->iommu); +err_put_group: + iommu_group_put(iommu->group); err_unprepare_clocks: clk_bulk_unprepare(iommu->num_clocks, iommu->clocks); return err; From 7868805969942a424118b7da5233946e53ce69be Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 29 Mar 2018 12:24:52 +0100 Subject: [PATCH 53/53] iommu/io-pgtable-arm: Avoid warning with 32-bit phys_addr_t It's not entirely unreasonable for io-pgtable-arm to be built for configurations with 32-bit phys_addr_t, where the compiler rightly raises a warning about the 36-bit shift. That particular code path should never actually *run* on those systems, but we still want it to compile cleanly, which is easily done by using an unambiguous u64 as the intermediate type instead. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index a5be4c92c5c8..47b64d3b833f 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -211,7 +211,7 @@ static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte, struct arm_lpae_io_pgtable *data) { - phys_addr_t paddr = pte & ARM_LPAE_PTE_ADDR_MASK; + u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; if (data->pg_shift < 16) return paddr;