bcachefs: Make io timers less buggy

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/clock.c

@@ -18,6 +18,14 @@ void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer)
 	size_t i;
 
 	spin_lock(&clock->timer_lock);
+
+	if (time_after_eq((unsigned long) atomic_long_read(&clock->now),
+			  timer->expire)) {
+		spin_unlock(&clock->timer_lock);
+		timer->fn(timer);
+		return;
+	}
+
 	for (i = 0; i < clock->timers.used; i++)
 		if (clock->timers.data[i] == timer)
 			goto out;
@@ -135,28 +143,33 @@ static struct io_timer *get_expired_timer(struct io_clock *clock,
 	return ret;
 }
 
-void __bch2_increment_clock(struct io_clock *clock)
+void __bch2_increment_clock(struct io_clock *clock, unsigned sectors)
 {
 	struct io_timer *timer;
-	unsigned long now;
-	unsigned sectors;
-
-	/* Buffer up one megabyte worth of IO in the percpu counter */
-	preempt_disable();
-
-	if (this_cpu_read(*clock->pcpu_buf) < IO_CLOCK_PCPU_SECTORS) {
-		preempt_enable();
-		return;
-	}
-
-	sectors = this_cpu_xchg(*clock->pcpu_buf, 0);
-	preempt_enable();
-	now = atomic_long_add_return(sectors, &clock->now);
+	unsigned long now = atomic_long_add_return(sectors, &clock->now);
 
 	while ((timer = get_expired_timer(clock, now)))
 		timer->fn(timer);
 }
 
+ssize_t bch2_io_timers_show(struct io_clock *clock, char *buf)
+{
+	struct printbuf out = _PBUF(buf, PAGE_SIZE);
+	unsigned long now;
+	unsigned i;
+
+	spin_lock(&clock->timer_lock);
+	now = atomic_long_read(&clock->now);
+
+	for (i = 0; i < clock->timers.used; i++)
+		pr_buf(&out, "%pf:\t%li\n",
+		       clock->timers.data[i]->fn,
+		       clock->timers.data[i]->expire - now);
+	spin_unlock(&clock->timer_lock);
+
+	return out.pos - buf;
+}
+
 void bch2_io_clock_exit(struct io_clock *clock)
 {
 	free_heap(&clock->timers);
@@ -168,6 +181,8 @@ int bch2_io_clock_init(struct io_clock *clock)
 	atomic_long_set(&clock->now, 0);
 	spin_lock_init(&clock->timer_lock);
 
+	clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus();
+
 	clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf);
 	if (!clock->pcpu_buf)
 		return -ENOMEM;

fs/bcachefs/clock.h

@@ -7,7 +7,7 @@ void bch2_io_timer_del(struct io_clock *, struct io_timer *);
 void bch2_kthread_io_clock_wait(struct io_clock *, unsigned long,
 				unsigned long);
 
-void __bch2_increment_clock(struct io_clock *);
+void __bch2_increment_clock(struct io_clock *, unsigned);
 
 static inline void bch2_increment_clock(struct bch_fs *c, unsigned sectors,
 					int rw)
@@ -16,7 +16,7 @@ static inline void bch2_increment_clock(struct bch_fs *c, unsigned sectors,
 
 	if (unlikely(this_cpu_add_return(*clock->pcpu_buf, sectors) >=
 		   IO_CLOCK_PCPU_SECTORS))
-		__bch2_increment_clock(clock);
+		__bch2_increment_clock(clock, this_cpu_xchg(*clock->pcpu_buf, 0));
 }
 
 void bch2_io_clock_schedule_timeout(struct io_clock *, unsigned long);
@@ -30,6 +30,8 @@ void bch2_io_clock_schedule_timeout(struct io_clock *, unsigned long);
 	__ret;								\
 })
 
+ssize_t bch2_io_timers_show(struct io_clock *, char *);
+
 void bch2_io_clock_exit(struct io_clock *);
 int bch2_io_clock_init(struct io_clock *);
 

fs/bcachefs/clock_types.h

@@ -28,6 +28,7 @@ typedef HEAP(struct io_timer *)	io_timer_heap;
 struct io_clock {
 	atomic_long_t		now;
 	u16 __percpu		*pcpu_buf;
+	unsigned		max_slop;
 
 	spinlock_t		timer_lock;
 	io_timer_heap		timers;

fs/bcachefs/sysfs.c

@@ -18,6 +18,7 @@
 #include "btree_update_interior.h"
 #include "btree_gc.h"
 #include "buckets.h"
+#include "clock.h"
 #include "disk_groups.h"
 #include "ec.h"
 #include "inode.h"
@@ -198,6 +199,9 @@ rw_attribute(pd_controllers_update_seconds);
 read_attribute(meta_replicas_have);
 read_attribute(data_replicas_have);
 
+read_attribute(io_timers_read);
+read_attribute(io_timers_write);
+
 #ifdef CONFIG_BCACHEFS_TESTS
 write_attribute(perf_test);
 #endif /* CONFIG_BCACHEFS_TESTS */
@@ -404,6 +408,11 @@ SHOW(bch2_fs)
 	if (attr == &sysfs_new_stripes)
 		return bch2_new_stripes(c, buf);
 
+	if (attr == &sysfs_io_timers_read)
+		return bch2_io_timers_show(&c->io_clock[READ], buf);
+	if (attr == &sysfs_io_timers_write)
+		return bch2_io_timers_show(&c->io_clock[WRITE], buf);
+
 #define BCH_DEBUG_PARAM(name, description) sysfs_print(name, c->name);
 	BCH_DEBUG_PARAMS()
 #undef BCH_DEBUG_PARAM
@@ -581,6 +590,9 @@ struct attribute *bch2_fs_internal_files[] = {
 
 	&sysfs_new_stripes,
 
+	&sysfs_io_timers_read,
+	&sysfs_io_timers_write,
+
 	&sysfs_internal_uuid,
 
 #define BCH_DEBUG_PARAM(name, description) &sysfs_##name,