Detailed benchmark outputs for functions

This patch adds an option to get detailed benchmark output for
functions.  Invoking the benchmark with 'make DETAILED=1 bench' causes
each benchmark program to store a mean execution time for each input
it works on.  This is useful to give a more comprehensive picture of
performance of functions compared to just the single mean figure.

ChangeLog

@@ -1,5 +1,15 @@
 2014-03-29  Siddhesh Poyarekar  <siddhesh@redhat.com>
 
+	* benchtests/Makefile (DETAILED_OPT): New make option.
+	(bench-func): Run benchmark program with -d if DETAILED_OPT is
+	set.
+	* benchtests/bench-skeleton.c: Include stdbool.h.
+	(main): Store and print timings per input.
+	* benchtests/scripts/bench.py (STRUCT_TEMPLATE): Add timing
+	member to each argument value.
+	(EPILOGUE): Define new macros RESULT and RESULT_ACCUM.
+	(_print_arg_data): Initialize per-input timing to 0.
+
 	* benchtests/Makefile (timing-type): New binary.
 	(bench-clean): Also remove bench-timing-type.
 	(bench): New target for timing-type.

benchtests/Makefile

@@ -86,6 +86,12 @@ ifdef USE_CLOCK_GETTIME
 CPPFLAGS-nonlib += -DUSE_CLOCK_GETTIME
 endif
 
+DETAILED_OPT :=
+
+ifdef DETAILED
+DETAILED_OPT := -d
+endif
+
 # This makes sure CPPFLAGS-nonlib and CFLAGS-nonlib are passed
 # for all these modules.
 cpp-srcs-left := $(binaries-benchset:=.c) $(binaries-bench:=.c)
@@ -126,7 +132,7 @@ bench-func: $(binaries-bench)
 	    echo ","; \
 	  fi; \
 	  echo "Running $${run}" >&2; \
-	  $(run-bench); \
+	  $(run-bench) $(DETAILED_OPT); \
 	done; \
 	echo "  }"; \
 	echo "}"; } > $(objpfx)bench.out-tmp; \

benchtests/bench-skeleton.c

@@ -18,6 +18,7 @@
 
 #include <string.h>
 #include <stdint.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <time.h>
 #include <inttypes.h>
@@ -48,6 +49,10 @@ main (int argc, char **argv)
   unsigned long i, k;
   struct timespec runtime;
   timing_t start, end;
+  bool detailed = false;
+
+  if (argc == 2 && !strcmp (argv[1], "-d"))
+    detailed = true;
 
   startup();
 
@@ -72,6 +77,7 @@ main (int argc, char **argv)
 
       double d_total_i = 0;
       timing_t total = 0, max = 0, min = 0x7fffffffffffffff;
+      int64_t c = 0;
       while (1)
 	{
 	  for (i = 0; i < NUM_SAMPLES (v); i++)
@@ -91,8 +97,13 @@ main (int argc, char **argv)
 		min = cur;
 
 	      TIMING_ACCUM (total, cur);
+	      /* Accumulate timings for the value.  In the end we will divide
+	         by the total iterations.  */
+	      RESULT_ACCUM (cur, v, i, c * iters, (c + 1) * iters);
+
 	      d_total_i += iters;
 	    }
+	  c++;
 	  struct timespec curtime;
 
 	  memset (&curtime, 0, sizeof (curtime));
@@ -114,6 +125,17 @@ main (int argc, char **argv)
 	      d_total_s, d_total_i, max / d_iters, min / d_iters,
 	      d_total_s / d_total_i);
 
+      if (detailed)
+	{
+	  printf (",\n\"timings\": [");
+	  for (int i = 0; i < NUM_SAMPLES (v); i++)
+	    {
+	      if (i > 0)
+		putc (',', stdout);
+	      printf ("%g", RESULT (v, i));
+	    }
+	  puts ("]");
+	}
       puts ("}");
     }
 

benchtests/scripts/bench.py

@@ -50,6 +50,7 @@ STRUCT_TEMPLATE = '''
 struct args
 {
 %(args)s
+  double timing;
 };
 
 struct _variants
@@ -80,6 +81,9 @@ struct _variants variants[%(num_variants)d] = {
 
 # Epilogue for the generated source file.
 EPILOGUE = '''
+#define RESULT(__v, __i) (variants[(__v)].in[(__i)].timing)
+#define RESULT_ACCUM(r, v, i, old, new) \\
+        ((RESULT ((v), (i))) = (RESULT ((v), (i)) * (old) + (r)) / ((new) + 1))
 #define BENCH_FUNC(i, j) ({%(getret)s CALL_BENCH_FUNC (i, j);})
 #define FUNCNAME "%(func)s"
 #include "bench-skeleton.c"'''
@@ -168,7 +172,7 @@ def _print_arg_data(func, directives, all_vals):
     # Now print the values.
     variants = []
     for (k, vals), i in zip(all_vals.items(), itertools.count()):
-        out = ['  {%s},' % v for v in vals]
+        out = ['  {%s, 0},' % v for v in vals]
 
         # Members for the variants structure list that we will
         # print later.