// SPDX-License-Identifier: GPL-2.0 #define CREATE_TRACE_POINTS #include <trace/events/mmap_lock.h> #include <linux/mm.h> #include <linux/cgroup.h> #include <linux/memcontrol.h> #include <linux/mmap_lock.h> #include <linux/mutex.h> #include <linux/percpu.h> #include <linux/rcupdate.h> #include <linux/smp.h> #include <linux/trace_events.h> #include <linux/local_lock.h> EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locking); EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_returned); EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released); #ifdef CONFIG_MEMCG /* * Our various events all share the same buffer (because we don't want or need * to allocate a set of buffers *per event type*), so we need to protect against * concurrent _reg() and _unreg() calls, and count how many _reg() calls have * been made. */ static DEFINE_MUTEX(reg_lock); static int reg_refcount; /* Protected by reg_lock. */ /* * Size of the buffer for memcg path names. Ignoring stack trace support, * trace_events_hist.c uses MAX_FILTER_STR_VAL for this, so we also use it. */ #define MEMCG_PATH_BUF_SIZE MAX_FILTER_STR_VAL /* * How many contexts our trace events might be called in: normal, softirq, irq, * and NMI. */ #define CONTEXT_COUNT 4 struct memcg_path { local_lock_t lock; char __rcu *buf; local_t buf_idx; }; static DEFINE_PER_CPU(struct memcg_path, memcg_paths) = { .lock = INIT_LOCAL_LOCK(lock), .buf_idx = LOCAL_INIT(0), }; static char **tmp_bufs; /* Called with reg_lock held. */ static void free_memcg_path_bufs(void) { struct memcg_path *memcg_path; int cpu; char **old = tmp_bufs; for_each_possible_cpu(cpu) { memcg_path = per_cpu_ptr(&memcg_paths, cpu); *(old++) = rcu_dereference_protected(memcg_path->buf, lockdep_is_held(®_lock)); rcu_assign_pointer(memcg_path->buf, NULL); } /* Wait for inflight memcg_path_buf users to finish. */ synchronize_rcu(); old = tmp_bufs; for_each_possible_cpu(cpu) { kfree(*(old++)); } kfree(tmp_bufs); tmp_bufs = NULL; } int trace_mmap_lock_reg(void) { int cpu; char *new; mutex_lock(®_lock); /* If the refcount is going 0->1, proceed with allocating buffers. */ if (reg_refcount++) goto out; tmp_bufs = kmalloc_array(num_possible_cpus(), sizeof(*tmp_bufs), GFP_KERNEL); if (tmp_bufs == NULL) goto out_fail; for_each_possible_cpu(cpu) { new = kmalloc(MEMCG_PATH_BUF_SIZE * CONTEXT_COUNT, GFP_KERNEL); if (new == NULL) goto out_fail_free; rcu_assign_pointer(per_cpu_ptr(&memcg_paths, cpu)->buf, new); /* Don't need to wait for inflights, they'd have gotten NULL. */ } out: mutex_unlock(®_lock); return 0; out_fail_free: free_memcg_path_bufs(); out_fail: /* Since we failed, undo the earlier ref increment. */ --reg_refcount; mutex_unlock(®_lock); return -ENOMEM; } void trace_mmap_lock_unreg(void) { mutex_lock(®_lock); /* If the refcount is going 1->0, proceed with freeing buffers. */ if (--reg_refcount) goto out; free_memcg_path_bufs(); out: mutex_unlock(®_lock); } static inline char *get_memcg_path_buf(void) { struct memcg_path *memcg_path = this_cpu_ptr(&memcg_paths); char *buf; int idx; rcu_read_lock(); buf = rcu_dereference(memcg_path->buf); if (buf == NULL) { rcu_read_unlock(); return NULL; } idx = local_add_return(MEMCG_PATH_BUF_SIZE, &memcg_path->buf_idx) - MEMCG_PATH_BUF_SIZE; return &buf[idx]; } static inline void put_memcg_path_buf(void) { local_sub(MEMCG_PATH_BUF_SIZE, &this_cpu_ptr(&memcg_paths)->buf_idx); rcu_read_unlock(); } #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \ do { \ const char *memcg_path; \ local_lock(&memcg_paths.lock); \ memcg_path = get_mm_memcg_path(mm); \ trace_mmap_lock_##type(mm, \ memcg_path != NULL ? memcg_path : "", \ ##__VA_ARGS__); \ if (likely(memcg_path != NULL)) \ put_memcg_path_buf(); \ local_unlock(&memcg_paths.lock); \ } while (0) #else /* !CONFIG_MEMCG */ int trace_mmap_lock_reg(void) { return 0; } void trace_mmap_lock_unreg(void) { } #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \ trace_mmap_lock_##type(mm, "", ##__VA_ARGS__) #endif /* CONFIG_MEMCG */ #ifdef CONFIG_TRACING #ifdef CONFIG_MEMCG /* * Write the given mm_struct's memcg path to a percpu buffer, and return a * pointer to it. If the path cannot be determined, or no buffer was available * (because the trace event is being unregistered), NULL is returned. * * Note: buffers are allocated per-cpu to avoid locking, so preemption must be * disabled by the caller before calling us, and re-enabled only after the * caller is done with the pointer. * * The caller must call put_memcg_path_buf() once the buffer is no longer * needed. This must be done while preemption is still disabled. */ static const char *get_mm_memcg_path(struct mm_struct *mm) { char *buf = NULL; struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); if (memcg == NULL) goto out; if (unlikely(memcg->css.cgroup == NULL)) goto out_put; buf = get_memcg_path_buf(); if (buf == NULL) goto out_put; cgroup_path(memcg->css.cgroup, buf, MEMCG_PATH_BUF_SIZE); out_put: css_put(&memcg->css); out: return buf; } #endif /* CONFIG_MEMCG */ /* * Trace calls must be in a separate file, as otherwise there's a circular * dependency between linux/mmap_lock.h and trace/events/mmap_lock.h. */ void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write) { TRACE_MMAP_LOCK_EVENT(start_locking, mm, write); } EXPORT_SYMBOL(__mmap_lock_do_trace_start_locking); void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write, bool success) { TRACE_MMAP_LOCK_EVENT(acquire_returned, mm, write, success); } EXPORT_SYMBOL(__mmap_lock_do_trace_acquire_returned); void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write) { TRACE_MMAP_LOCK_EVENT(released, mm, write); } EXPORT_SYMBOL(__mmap_lock_do_trace_released); #endif /* CONFIG_TRACING */