korg/linux
0
0
Fork 0
mirror of https://mirrors.bfsu.edu.cn/git/linux.git synced 2024-11-22 19:54:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

dma-mapping updates for Linux 6.13

Browse Source 
- improve the DMA API tracing code (Sean Anderson)
  - misc cleanups (Christoph Hellwig, Sui Jingfeng)
  - fix pointer abuse when finding the shared DMA pool (Geert Uytterhoeven)
  - fix a deadlock in dma-debug (Levi Yun)
 -----BEGIN PGP SIGNATURE-----
 
 iQI/BAABCgApFiEEgdbnc3r/njty3Iq9D55TZVIEUYMFAmc8xN8LHGhjaEBsc3Qu
 ZGUACgkQD55TZVIEUYNwEBAAtd0zTiNuEUklY6YtZ7l/Zaudibmq1klHLGAQZEa9
 J4P2zzJ6xTkUblq/aVmFUQmf+vuuszjHIrrXnL3tAulSQKxS5Zj3Cci4cW4IAfBn
 GXB3OTR2lgXSk+8sulgiwc1AA8xgIFJJgZDTni1WdiW9LwLvUyYI1XNVAwCYOM2J
 HS2QxIySm3eg23F5bRz+Xl3LQlWYlHkMHryqKloHWIqchmVpYlYbj7uBMjAH4FKz
 l3zhd9pZSp9w5NNCp2Y/d81XdOUSjcYSR1gUotLzmW0Sj3YjnKXKdjjlPrj3zimb
 9EhgdalnpVrJ4Nr7MmpSUEbTVs+hBjXDoxTnnBRlKEl5aIKqceCrSBvoP70ygbkf
 KRqNS4ZxKe59cfnWAZQVcg8g01TetCoJR6QyGaoTE9Lz+9cPl2xAwyFmcYN2w/Cp
 qs0ZEFiNpqLAN5zwR/Pakz5YgIA/3N5MW0d9X9yEH9l4+HUMxWIF/qvThBSsGswT
 EmVUQqPpEzGJrcNYgC1UsEBltGmle02BwcoFEdMr7bzldW7yIpoDEOkKkBM3JFF9
 vgkpAkZGA5j4VMSkSwOrhi1rI0XAoImtJeM0wqhLtpXgQDjrMd3DaW6by6uUeH5x
 DcXf6qVOAsB04je9JkHh9I4BXVrWC01MSgFdjfQRl9gktn7970YFswG4ksYAwxU6
 xHQ=
 =ivZc
 -----END PGP SIGNATURE-----

Merge tag 'dma-mapping-6.13-2024-11-19' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping updates from Christoph Hellwig:

 - improve the DMA API tracing code (Sean Anderson)

 - misc cleanups (Christoph Hellwig, Sui Jingfeng)

 - fix pointer abuse when finding the shared DMA pool (Geert
   Uytterhoeven)

 - fix a deadlock in dma-debug (Levi Yun)

* tag 'dma-mapping-6.13-2024-11-19' of git://git.infradead.org/users/hch/dma-mapping:
  dma-mapping: save base/size instead of pointer to shared DMA pool
  dma-mapping: fix swapped dir/flags arguments to trace_dma_alloc_sgt_err
  dma-mapping: drop unneeded includes from dma-mapping.h
  dma-mapping: trace more error paths
  dma-mapping: use trace_dma_alloc for dma_alloc* instead of using trace_dma_map
  dma-mapping: trace dma_alloc/free direction
  dma-mapping: use macros to define events in a class
  dma-mapping: remove an outdated comment from dma-map-ops.h
  dma-debug: remove DMA_API_DEBUG_SG
  dma-debug: store a phys_addr_t in struct dma_debug_entry
  dma-debug: fix a possible deadlock on radix_lock
This commit is contained in:
Linus Torvalds 2024-11-21 11:28:39 -08:00
commit 51ae62a12c
8 changed files with 242 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/powerpc/platforms/pseries/svm.c
View File

@ -8,6 +8,7 @@
#include <linux/mm.h>
#include <linux/memblock.h>
#include <linux/mem_encrypt.h>
#include <linux/cc_platform.h>
#include <asm/machdep.h>
#include <asm/svm.h>

2
include/linux/dma-map-ops.h
View File

@ -242,7 +242,7 @@ static inline bool dev_is_dma_coherent(struct device *dev)
{
return true;
}
#endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */
#endif
static inline void dma_reset_need_sync(struct device *dev)
{

4
include/linux/dma-mapping.h
View File

@ -2,15 +2,11 @@
#ifndef _LINUX_DMA_MAPPING_H
#define _LINUX_DMA_MAPPING_H
#include <linux/cache.h>
#include <linux/sizes.h>
#include <linux/string.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/dma-direction.h>
#include <linux/scatterlist.h>
#include <linux/bug.h>
#include <linux/mem_encrypt.h>
/**
* List of possible attributes associated with a DMA mapping. The semantics

213
include/trace/events/dma.h
View File

@ -65,15 +65,14 @@ DECLARE_EVENT_CLASS(dma_map,
decode_dma_attrs(__entry->attrs))
);
DEFINE_EVENT(dma_map, dma_map_page,
TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr,
size_t size, enum dma_data_direction dir, unsigned long attrs),
TP_ARGS(dev, phys_addr, dma_addr, size, dir, attrs));
#define DEFINE_MAP_EVENT(name) \
DEFINE_EVENT(dma_map, name, \
TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr, \
size_t size, enum dma_data_direction dir, unsigned long attrs), \
TP_ARGS(dev, phys_addr, dma_addr, size, dir, attrs))
DEFINE_EVENT(dma_map, dma_map_resource,
TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr,
size_t size, enum dma_data_direction dir, unsigned long attrs),
TP_ARGS(dev, phys_addr, dma_addr, size, dir, attrs));
DEFINE_MAP_EVENT(dma_map_page);
DEFINE_MAP_EVENT(dma_map_resource);
DECLARE_EVENT_CLASS(dma_unmap,
TP_PROTO(struct device *dev, dma_addr_t addr, size_t size,
@ -104,20 +103,20 @@ DECLARE_EVENT_CLASS(dma_unmap,
decode_dma_attrs(__entry->attrs))
);
DEFINE_EVENT(dma_unmap, dma_unmap_page,
TP_PROTO(struct device *dev, dma_addr_t addr, size_t size,
enum dma_data_direction dir, unsigned long attrs),
TP_ARGS(dev, addr, size, dir, attrs));
#define DEFINE_UNMAP_EVENT(name) \
DEFINE_EVENT(dma_unmap, name, \
TP_PROTO(struct device *dev, dma_addr_t addr, size_t size, \
enum dma_data_direction dir, unsigned long attrs), \
TP_ARGS(dev, addr, size, dir, attrs))
DEFINE_EVENT(dma_unmap, dma_unmap_resource,
TP_PROTO(struct device *dev, dma_addr_t addr, size_t size,
enum dma_data_direction dir, unsigned long attrs),
TP_ARGS(dev, addr, size, dir, attrs));
DEFINE_UNMAP_EVENT(dma_unmap_page);
DEFINE_UNMAP_EVENT(dma_unmap_resource);
TRACE_EVENT(dma_alloc,
DECLARE_EVENT_CLASS(dma_alloc_class,
TP_PROTO(struct device *dev, void *virt_addr, dma_addr_t dma_addr,
size_t size, gfp_t flags, unsigned long attrs),
TP_ARGS(dev, virt_addr, dma_addr, size, flags, attrs),
size_t size, enum dma_data_direction dir, gfp_t flags,
unsigned long attrs),
TP_ARGS(dev, virt_addr, dma_addr, size, dir, flags, attrs),
TP_STRUCT__entry(
__string(device, dev_name(dev))
@ -125,6 +124,7 @@ TRACE_EVENT(dma_alloc,
__field(u64, dma_addr)
__field(size_t, size)
__field(gfp_t, flags)
__field(enum dma_data_direction, dir)
__field(unsigned long, attrs)
),
@ -137,8 +137,9 @@ TRACE_EVENT(dma_alloc,
__entry->attrs = attrs;
),
TP_printk("%s dma_addr=%llx size=%zu virt_addr=%p flags=%s attrs=%s",
TP_printk("%s dir=%s dma_addr=%llx size=%zu virt_addr=%p flags=%s attrs=%s",
__get_str(device),
decode_dma_data_direction(__entry->dir),
__entry->dma_addr,
__entry->size,
__entry->virt_addr,
@ -146,16 +147,69 @@ TRACE_EVENT(dma_alloc,
decode_dma_attrs(__entry->attrs))
);
TRACE_EVENT(dma_free,
#define DEFINE_ALLOC_EVENT(name) \
DEFINE_EVENT(dma_alloc_class, name, \
TP_PROTO(struct device *dev, void *virt_addr, dma_addr_t dma_addr, \
size_t size, enum dma_data_direction dir, gfp_t flags, \
unsigned long attrs), \
TP_ARGS(dev, virt_addr, dma_addr, size, dir, flags, attrs))
DEFINE_ALLOC_EVENT(dma_alloc);
DEFINE_ALLOC_EVENT(dma_alloc_pages);
DEFINE_ALLOC_EVENT(dma_alloc_sgt_err);
TRACE_EVENT(dma_alloc_sgt,
TP_PROTO(struct device *dev, struct sg_table *sgt, size_t size,
enum dma_data_direction dir, gfp_t flags, unsigned long attrs),
TP_ARGS(dev, sgt, size, dir, flags, attrs),
TP_STRUCT__entry(
__string(device, dev_name(dev))
__dynamic_array(u64, phys_addrs, sgt->orig_nents)
__field(u64, dma_addr)
__field(size_t, size)
__field(enum dma_data_direction, dir)
__field(gfp_t, flags)
__field(unsigned long, attrs)
),
TP_fast_assign(
struct scatterlist *sg;
int i;
__assign_str(device);
for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
((u64 *)__get_dynamic_array(phys_addrs))[i] = sg_phys(sg);
__entry->dma_addr = sg_dma_address(sgt->sgl);
__entry->size = size;
__entry->dir = dir;
__entry->flags = flags;
__entry->attrs = attrs;
),
TP_printk("%s dir=%s dma_addr=%llx size=%zu phys_addrs=%s flags=%s attrs=%s",
__get_str(device),
decode_dma_data_direction(__entry->dir),
__entry->dma_addr,
__entry->size,
__print_array(__get_dynamic_array(phys_addrs),
__get_dynamic_array_len(phys_addrs) /
sizeof(u64), sizeof(u64)),
show_gfp_flags(__entry->flags),
decode_dma_attrs(__entry->attrs))
);
DECLARE_EVENT_CLASS(dma_free_class,
TP_PROTO(struct device *dev, void *virt_addr, dma_addr_t dma_addr,
size_t size, unsigned long attrs),
TP_ARGS(dev, virt_addr, dma_addr, size, attrs),
size_t size, enum dma_data_direction dir, unsigned long attrs),
TP_ARGS(dev, virt_addr, dma_addr, size, dir, attrs),
TP_STRUCT__entry(
__string(device, dev_name(dev))
__field(void *, virt_addr)
__field(u64, dma_addr)
__field(size_t, size)
__field(enum dma_data_direction, dir)
__field(unsigned long, attrs)
),
@ -164,17 +218,63 @@ TRACE_EVENT(dma_free,
__entry->virt_addr = virt_addr;
__entry->dma_addr = dma_addr;
__entry->size = size;
__entry->dir = dir;
__entry->attrs = attrs;
),
TP_printk("%s dma_addr=%llx size=%zu virt_addr=%p attrs=%s",
TP_printk("%s dir=%s dma_addr=%llx size=%zu virt_addr=%p attrs=%s",
__get_str(device),
decode_dma_data_direction(__entry->dir),
__entry->dma_addr,
__entry->size,
__entry->virt_addr,
decode_dma_attrs(__entry->attrs))
);
#define DEFINE_FREE_EVENT(name) \
DEFINE_EVENT(dma_free_class, name, \
TP_PROTO(struct device *dev, void *virt_addr, dma_addr_t dma_addr, \
size_t size, enum dma_data_direction dir, unsigned long attrs), \
TP_ARGS(dev, virt_addr, dma_addr, size, dir, attrs))
DEFINE_FREE_EVENT(dma_free);
DEFINE_FREE_EVENT(dma_free_pages);
TRACE_EVENT(dma_free_sgt,
TP_PROTO(struct device *dev, struct sg_table *sgt, size_t size,
enum dma_data_direction dir),
TP_ARGS(dev, sgt, size, dir),
TP_STRUCT__entry(
__string(device, dev_name(dev))
__dynamic_array(u64, phys_addrs, sgt->orig_nents)
__field(u64, dma_addr)
__field(size_t, size)
__field(enum dma_data_direction, dir)
),
TP_fast_assign(
struct scatterlist *sg;
int i;
__assign_str(device);
for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
((u64 *)__get_dynamic_array(phys_addrs))[i] = sg_phys(sg);
__entry->dma_addr = sg_dma_address(sgt->sgl);
__entry->size = size;
__entry->dir = dir;
),
TP_printk("%s dir=%s dma_addr=%llx size=%zu phys_addrs=%s",
__get_str(device),
decode_dma_data_direction(__entry->dir),
__entry->dma_addr,
__entry->size,
__print_array(__get_dynamic_array(phys_addrs),
__get_dynamic_array_len(phys_addrs) /
sizeof(u64), sizeof(u64)))
);
TRACE_EVENT(dma_map_sg,
TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents,
int ents, enum dma_data_direction dir, unsigned long attrs),
@ -221,6 +321,41 @@ TRACE_EVENT(dma_map_sg,
decode_dma_attrs(__entry->attrs))
);
TRACE_EVENT(dma_map_sg_err,
TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents,
int err, enum dma_data_direction dir, unsigned long attrs),
TP_ARGS(dev, sgl, nents, err, dir, attrs),
TP_STRUCT__entry(
__string(device, dev_name(dev))
__dynamic_array(u64, phys_addrs, nents)
__field(int, err)
__field(enum dma_data_direction, dir)
__field(unsigned long, attrs)
),
TP_fast_assign(
struct scatterlist *sg;
int i;
__assign_str(device);
for_each_sg(sgl, sg, nents, i)
((u64 *)__get_dynamic_array(phys_addrs))[i] = sg_phys(sg);
__entry->err = err;
__entry->dir = dir;
__entry->attrs = attrs;
),
TP_printk("%s dir=%s dma_addrs=%s err=%d attrs=%s",
__get_str(device),
decode_dma_data_direction(__entry->dir),
__print_array(__get_dynamic_array(phys_addrs),
__get_dynamic_array_len(phys_addrs) /
sizeof(u64), sizeof(u64)),
__entry->err,
decode_dma_attrs(__entry->attrs))
);
TRACE_EVENT(dma_unmap_sg,
TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents,
enum dma_data_direction dir, unsigned long attrs),
@ -279,15 +414,14 @@ DECLARE_EVENT_CLASS(dma_sync_single,
__entry->size)
);
DEFINE_EVENT(dma_sync_single, dma_sync_single_for_cpu,
TP_PROTO(struct device *dev, dma_addr_t dma_addr, size_t size,
enum dma_data_direction dir),
TP_ARGS(dev, dma_addr, size, dir));
#define DEFINE_SYNC_SINGLE_EVENT(name) \
DEFINE_EVENT(dma_sync_single, name, \
TP_PROTO(struct device *dev, dma_addr_t dma_addr, size_t size, \
enum dma_data_direction dir), \
TP_ARGS(dev, dma_addr, size, dir))
DEFINE_EVENT(dma_sync_single, dma_sync_single_for_device,
TP_PROTO(struct device *dev, dma_addr_t dma_addr, size_t size,
enum dma_data_direction dir),
TP_ARGS(dev, dma_addr, size, dir));
DEFINE_SYNC_SINGLE_EVENT(dma_sync_single_for_cpu);
DEFINE_SYNC_SINGLE_EVENT(dma_sync_single_for_device);
DECLARE_EVENT_CLASS(dma_sync_sg,
TP_PROTO(struct device *dev, struct scatterlist *sgl, int nents,
@ -326,15 +460,14 @@ DECLARE_EVENT_CLASS(dma_sync_sg,
sizeof(unsigned int), sizeof(unsigned int)))
);
DEFINE_EVENT(dma_sync_sg, dma_sync_sg_for_cpu,
TP_PROTO(struct device *dev, struct scatterlist *sg, int nents,
enum dma_data_direction dir),
TP_ARGS(dev, sg, nents, dir));
#define DEFINE_SYNC_SG_EVENT(name) \
DEFINE_EVENT(dma_sync_sg, name, \
TP_PROTO(struct device *dev, struct scatterlist *sg, int nents, \
enum dma_data_direction dir), \
TP_ARGS(dev, sg, nents, dir))
DEFINE_EVENT(dma_sync_sg, dma_sync_sg_for_device,
TP_PROTO(struct device *dev, struct scatterlist *sg, int nents,
enum dma_data_direction dir),
TP_ARGS(dev, sg, nents, dir));
DEFINE_SYNC_SG_EVENT(dma_sync_sg_for_cpu);
DEFINE_SYNC_SG_EVENT(dma_sync_sg_for_device);
#endif /* _TRACE_DMA_H */

17
kernel/dma/Kconfig
View File

@ -260,23 +260,6 @@ config DMA_API_DEBUG
If unsure, say N.
config DMA_API_DEBUG_SG
bool "Debug DMA scatter-gather usage"
default y
depends on DMA_API_DEBUG
help
Perform extra checking that callers of dma_map_sg() have respected the
appropriate segment length/boundary limits for the given device when
preparing DMA scatterlists.
This is particularly likely to have been overlooked in cases where the
dma_map_sg() API is used for general bulk mapping of pages rather than
preparing literal scatter-gather descriptors, where there is a risk of
unexpected behaviour from DMA API implementations if the scatterlist
is technically out-of-spec.
If unsure, say N.
config DMA_MAP_BENCHMARK
bool "Enable benchmarking of streaming DMA mapping"
depends on DEBUG_FS

14
kernel/dma/coherent.c
View File

@ -330,7 +330,8 @@ int dma_init_global_coherent(phys_addr_t phys_addr, size_t size)
#include <linux/of_reserved_mem.h>
#ifdef CONFIG_DMA_GLOBAL_POOL
static struct reserved_mem *dma_reserved_default_memory __initdata;
static phys_addr_t dma_reserved_default_memory_base __initdata;
static phys_addr_t dma_reserved_default_memory_size __initdata;
#endif
static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev)
@ -376,9 +377,10 @@ static int __init rmem_dma_setup(struct reserved_mem *rmem)
#ifdef CONFIG_DMA_GLOBAL_POOL
if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {
WARN(dma_reserved_default_memory,
WARN(dma_reserved_default_memory_size,
"Reserved memory: region for default DMA coherent area is redefined\n");
dma_reserved_default_memory = rmem;
dma_reserved_default_memory_base = rmem->base;
dma_reserved_default_memory_size = rmem->size;
}
#endif
@ -391,10 +393,10 @@ static int __init rmem_dma_setup(struct reserved_mem *rmem)
#ifdef CONFIG_DMA_GLOBAL_POOL
static int __init dma_init_reserved_memory(void)
{
if (!dma_reserved_default_memory)
if (!dma_reserved_default_memory_size)
return -ENOMEM;
return dma_init_global_coherent(dma_reserved_default_memory->base,
dma_reserved_default_memory->size);
return dma_init_global_coherent(dma_reserved_default_memory_base,
dma_reserved_default_memory_size);
}
core_initcall(dma_init_reserved_memory);
#endif /* CONFIG_DMA_GLOBAL_POOL */

89
kernel/dma/debug.c
View File

@ -59,8 +59,7 @@ enum map_err_types {
* @direction: enum dma_data_direction
* @sg_call_ents: 'nents' from dma_map_sg
* @sg_mapped_ents: 'mapped_ents' from dma_map_sg
* @pfn: page frame of the start address
* @offset: offset of mapping relative to pfn
* @paddr: physical start address of the mapping
* @map_err_type: track whether dma_mapping_error() was checked
* @stack_len: number of backtrace entries in @stack_entries
* @stack_entries: stack of backtrace history
@ -74,8 +73,7 @@ struct dma_debug_entry {
int direction;
int sg_call_ents;
int sg_mapped_ents;
unsigned long pfn;
size_t offset;
phys_addr_t paddr;
enum map_err_types map_err_type;
#ifdef CONFIG_STACKTRACE
unsigned int stack_len;
@ -389,14 +387,6 @@ static void hash_bucket_del(struct dma_debug_entry *entry)
list_del(&entry->list);
}
static unsigned long long phys_addr(struct dma_debug_entry *entry)
{
if (entry->type == dma_debug_resource)
return __pfn_to_phys(entry->pfn) + entry->offset;
return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;
}
/*
* For each mapping (initial cacheline in the case of
* dma_alloc_coherent/dma_map_page, initial cacheline in each page of a
@ -428,8 +418,8 @@ static DEFINE_SPINLOCK(radix_lock);
static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
{
return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +
(entry->offset >> L1_CACHE_SHIFT);
return ((entry->paddr >> PAGE_SHIFT) << CACHELINE_PER_PAGE_SHIFT) +
(offset_in_page(entry->paddr) >> L1_CACHE_SHIFT);
}
static int active_cacheline_read_overlap(phys_addr_t cln)
@ -538,11 +528,11 @@ void debug_dma_dump_mappings(struct device *dev)
if (!dev || dev == entry->dev) {
cln = to_cacheline_number(entry);
dev_info(entry->dev,
"%s idx %d P=%llx N=%lx D=%llx L=%llx cln=%pa %s %s\n",
"%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n",
type2name[entry->type], idx,
phys_addr(entry), entry->pfn,
entry->dev_addr, entry->size,
&cln, dir2name[entry->direction],
&entry->paddr, entry->dev_addr,
entry->size, &cln,
dir2name[entry->direction],
maperr2str[entry->map_err_type]);
}
}
@ -569,13 +559,13 @@ static int dump_show(struct seq_file *seq, void *v)
list_for_each_entry(entry, &bucket->list, list) {
cln = to_cacheline_number(entry);
seq_printf(seq,
"%s %s %s idx %d P=%llx N=%lx D=%llx L=%llx cln=%pa %s %s\n",
"%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n",
dev_driver_string(entry->dev),
dev_name(entry->dev),
type2name[entry->type], idx,
phys_addr(entry), entry->pfn,
entry->dev_addr, entry->size,
&cln, dir2name[entry->direction],
&entry->paddr, entry->dev_addr,
entry->size, &cln,
dir2name[entry->direction],
maperr2str[entry->map_err_type]);
}
spin_unlock_irqrestore(&bucket->lock, flags);
@ -1003,16 +993,16 @@ static void check_unmap(struct dma_debug_entry *ref)
"[mapped as %s] [unmapped as %s]\n",
ref->dev_addr, ref->size,
type2name[entry->type], type2name[ref->type]);
} else if ((entry->type == dma_debug_coherent) &&
(phys_addr(ref) != phys_addr(entry))) {
} else if (entry->type == dma_debug_coherent &&
ref->paddr != entry->paddr) {
err_printk(ref->dev, entry, "device driver frees "
"DMA memory with different CPU address "
"[device address=0x%016llx] [size=%llu bytes] "
"[cpu alloc address=0x%016llx] "
"[cpu free address=0x%016llx]",
"[cpu alloc address=0x%pa] "
"[cpu free address=0x%pa]",
ref->dev_addr, ref->size,
phys_addr(entry),
phys_addr(ref));
&entry->paddr,
&ref->paddr);
}
if (ref->sg_call_ents && ref->type == dma_debug_sg &&
@ -1052,9 +1042,13 @@ static void check_unmap(struct dma_debug_entry *ref)
}
hash_bucket_del(entry);
dma_entry_free(entry);
put_hash_bucket(bucket, flags);
/*
* Free the entry outside of bucket_lock to avoid ABBA deadlocks
* between that and radix_lock.
*/
dma_entry_free(entry);
}
static void check_for_stack(struct device *dev,
@ -1169,7 +1163,6 @@ out:
static void check_sg_segment(struct device *dev, struct scatterlist *sg)
{
#ifdef CONFIG_DMA_API_DEBUG_SG
unsigned int max_seg = dma_get_max_seg_size(dev);
u64 start, end, boundary = dma_get_seg_boundary(dev);
@ -1190,7 +1183,6 @@ static void check_sg_segment(struct device *dev, struct scatterlist *sg)
if ((start ^ end) & ~boundary)
err_printk(dev, NULL, "mapping sg segment across boundary [start=0x%016llx] [end=0x%016llx] [boundary=0x%016llx]\n",
start, end, boundary);
#endif
}
void debug_dma_map_single(struct device *dev, const void *addr,
@ -1227,8 +1219,7 @@ void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
entry->dev = dev;
entry->type = dma_debug_single;
entry->pfn = page_to_pfn(page);
entry->offset = offset;
entry->paddr = page_to_phys(page);
entry->dev_addr = dma_addr;
entry->size = size;
entry->direction = direction;
@ -1323,8 +1314,7 @@ void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
entry->type = dma_debug_sg;
entry->dev = dev;
entry->pfn = page_to_pfn(sg_page(s));
entry->offset = s->offset;
entry->paddr = sg_phys(s);
entry->size = sg_dma_len(s);
entry->dev_addr = sg_dma_address(s);
entry->direction = direction;
@ -1370,8 +1360,7 @@ void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
struct dma_debug_entry ref = {
.type = dma_debug_sg,
.dev = dev,
.pfn = page_to_pfn(sg_page(s)),
.offset = s->offset,
.paddr = sg_phys(s),
.dev_addr = sg_dma_address(s),
.size = sg_dma_len(s),
.direction = dir,
@ -1410,16 +1399,12 @@ void debug_dma_alloc_coherent(struct device *dev, size_t size,
entry->type = dma_debug_coherent;
entry->dev = dev;
entry->offset = offset_in_page(virt);
entry->paddr = page_to_phys((is_vmalloc_addr(virt) ?
vmalloc_to_page(virt) : virt_to_page(virt)));
entry->size = size;
entry->dev_addr = dma_addr;
entry->direction = DMA_BIDIRECTIONAL;
if (is_vmalloc_addr(virt))
entry->pfn = vmalloc_to_pfn(virt);
else
entry->pfn = page_to_pfn(virt_to_page(virt));
add_dma_entry(entry, attrs);
}
@ -1429,7 +1414,6 @@ void debug_dma_free_coherent(struct device *dev, size_t size,
struct dma_debug_entry ref = {
.type = dma_debug_coherent,
.dev = dev,
.offset = offset_in_page(virt),
.dev_addr = dma_addr,
.size = size,
.direction = DMA_BIDIRECTIONAL,
@ -1439,10 +1423,8 @@ void debug_dma_free_coherent(struct device *dev, size_t size,
if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt))
return;
if (is_vmalloc_addr(virt))
ref.pfn = vmalloc_to_pfn(virt);
else
ref.pfn = page_to_pfn(virt_to_page(virt));
ref.paddr = page_to_phys((is_vmalloc_addr(virt) ?
vmalloc_to_page(virt) : virt_to_page(virt)));
if (unlikely(dma_debug_disabled()))
return;
@ -1465,8 +1447,7 @@ void debug_dma_map_resource(struct device *dev, phys_addr_t addr, size_t size,
entry->type = dma_debug_resource;
entry->dev = dev;
entry->pfn = PHYS_PFN(addr);
entry->offset = offset_in_page(addr);
entry->paddr = addr;
entry->size = size;
entry->dev_addr = dma_addr;
entry->direction = direction;
@ -1543,8 +1524,7 @@ void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
struct dma_debug_entry ref = {
.type = dma_debug_sg,
.dev = dev,
.pfn = page_to_pfn(sg_page(s)),
.offset = s->offset,
.paddr = sg_phys(s),
.dev_addr = sg_dma_address(s),
.size = sg_dma_len(s),
.direction = direction,
@ -1575,8 +1555,7 @@ void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
struct dma_debug_entry ref = {
.type = dma_debug_sg,
.dev = dev,
.pfn = page_to_pfn(sg_page(s)),
.offset = s->offset,
.paddr = sg_phys(sg),
.dev_addr = sg_dma_address(s),
.size = sg_dma_len(s),
.direction = direction,

37
kernel/dma/mapping.c
View File

@ -223,6 +223,7 @@ static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
debug_dma_map_sg(dev, sg, nents, ents, dir, attrs);
} else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&
ents != -EIO && ents != -EREMOTEIO)) {
trace_dma_map_sg_err(dev, sg, nents, ents, dir, attrs);
return -EIO;
}
@ -604,22 +605,29 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
if (WARN_ON_ONCE(flag & __GFP_COMP))
return NULL;
if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {
trace_dma_alloc(dev, cpu_addr, *dma_handle, size,
DMA_BIDIRECTIONAL, flag, attrs);
return cpu_addr;
}
/* let the implementation decide on the zone to allocate from: */
flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
if (dma_alloc_direct(dev, ops))
if (dma_alloc_direct(dev, ops)) {
cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
else if (use_dma_iommu(dev))
} else if (use_dma_iommu(dev)) {
cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs);
else if (ops->alloc)
} else if (ops->alloc) {
cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
else
} else {
trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,
attrs);
return NULL;
}
trace_dma_alloc(dev, cpu_addr, *dma_handle, size, flag, attrs);
trace_dma_alloc(dev, cpu_addr, *dma_handle, size, DMA_BIDIRECTIONAL,
flag, attrs);
debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs);
return cpu_addr;
}
@ -641,10 +649,11 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
*/
WARN_ON(irqs_disabled());
trace_dma_free(dev, cpu_addr, dma_handle, size, DMA_BIDIRECTIONAL,
attrs);
if (!cpu_addr)
return;
trace_dma_free(dev, cpu_addr, dma_handle, size, attrs);
debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
if (dma_alloc_direct(dev, ops))
dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
@ -683,9 +692,11 @@ struct page *dma_alloc_pages(struct device *dev, size_t size,
struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp);
if (page) {
trace_dma_map_page(dev, page_to_phys(page), *dma_handle, size,
dir, 0);
trace_dma_alloc_pages(dev, page_to_virt(page), *dma_handle,
size, dir, gfp, 0);
debug_dma_map_page(dev, page, 0, size, dir, *dma_handle, 0);
} else {
trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0);
}
return page;
}
@ -708,7 +719,7 @@ static void __dma_free_pages(struct device *dev, size_t size, struct page *page,
void dma_free_pages(struct device *dev, size_t size, struct page *page,
dma_addr_t dma_handle, enum dma_data_direction dir)
{
trace_dma_unmap_page(dev, dma_handle, size, dir, 0);
trace_dma_free_pages(dev, page_to_virt(page), dma_handle, size, dir, 0);
debug_dma_unmap_page(dev, dma_handle, size, dir);
__dma_free_pages(dev, size, page, dma_handle, dir);
}
@ -768,8 +779,10 @@ struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,
if (sgt) {
sgt->nents = 1;
trace_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs);
trace_dma_alloc_sgt(dev, sgt, size, dir, gfp, attrs);
debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs);
} else {
trace_dma_alloc_sgt_err(dev, NULL, 0, size, dir, gfp, attrs);
}
return sgt;
}
@ -787,7 +800,7 @@ static void free_single_sgt(struct device *dev, size_t size,
void dma_free_noncontiguous(struct device *dev, size_t size,
struct sg_table *sgt, enum dma_data_direction dir)
{
trace_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir, 0);
trace_dma_free_sgt(dev, sgt, size, dir);
debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir);
if (use_dma_iommu(dev))