math: Add more details to the test driver output.

Add start and end indicators that identify the test being run in the
verbose output.  Better identify the tests for max errors in the
summary output.  Count each exception checked for each test. Remove
double counting of tests for the check_<type> functions other than
check_float_internal. Rename print_max_error and
print_complex_max_error to check_max_error and check_complex_max_error
respectively since they have side effects.

Co-Authored-By: Carlos O'Donell <carlos@redhat.com>
Reviewed-By: Joseph Myers <josmyers@redhat.com>

math/libm-test-driver.c

@@ -1083,9 +1083,9 @@ struct test_Ff_b1_data
     = STR_CON3 (FUN, SUFF, TEST_SUFF) TEST_SUFF_STR;		\
   init_max_error (this_func, EXACT, TEST_COND_any_ibm128)
 #define END					\
-  print_max_error (this_func)
+  check_max_error (this_func)
 #define END_COMPLEX				\
-  print_complex_max_error (this_func)
+  check_complex_max_error (this_func)
 
 /* Run tests for a given function in all rounding modes.  */
 #define ALL_RM_TEST(FUNC, EXACT, ARRAY, LOOP_MACRO, END_MACRO, ...)	\

math/libm-test-support.c

@@ -112,6 +112,7 @@
 #include <argp.h>
 #include <errno.h>
 #include <string.h>
+#include <assert.h>
 
 /* This header defines func_ulps, func_real_ulps and func_imag_ulps
    arrays.  */
@@ -125,10 +126,13 @@ static FILE *ulps_file;		/* File to document difference.  */
 static int output_ulps;		/* Should ulps printed?  */
 static char *output_dir;	/* Directory where generated files will be written.  */
 
-static int noErrors;	/* number of errors */
+#define TEST_INPUT 1
-static int noTests;	/* number of tests (without testing exceptions) */
+#define TEST_MAXERROR 2
-static int noExcTests;	/* number of tests for exception flags */
+static int noErrors;	    /* number of errors */
-static int noErrnoTests;/* number of tests for errno values */
+static int noTests; 	    /* number of tests (without testing exceptions) */
+static int noMaxErrorTests; /* number of max error tests  */
+static int noExcTests;	    /* number of tests for exception flags */
+static int noErrnoTests;    /* number of tests for errno values */
 
 static unsigned int verbose;
 static int output_max_error;	/* Should the maximal errors printed?  */
@@ -299,9 +303,19 @@ print_screen_max_error (int ok)
 
 /* Update statistic counters.  */
 static void
-update_stats (int ok)
+update_stats (int ok, int testType)
 {
-  ++noTests;
+  switch (testType)
+    {
+	case TEST_INPUT:
+	  ++noTests;
+	  break;
+	case TEST_MAXERROR:
+	  ++noMaxErrorTests;
+	  break;
+	default:
+	  abort();
+    }
   if (!ok)
     ++noErrors;
 }
@@ -367,11 +381,30 @@ fpstack_test (const char *test_name)
 #endif
 }
 
+static void
+print_test_start (int test_num, const char *test_name, int test_type)
+{
+  if (print_screen (1))
+    printf ("--- Start of%s test # %d, named \"%s\" ---\n",
+            test_type == TEST_MAXERROR ? " max error" : "", test_num, test_name);
+}
 
+static void
+print_test_end (int test_num, const char *test_name, int test_type)
+{
+  if (print_screen (1))
+    printf ("--- End of%s test # %d, named \"%s\" ---\n",
+            test_type == TEST_MAXERROR ? " max error" : "", test_num, test_name);
+}
+
+/* This is a builtin test of overall max error.  */
 void
-print_max_error (const char *func_name)
+check_max_error (const char *func_name)
 {
   int ok = 0;
+  int thisTest = noMaxErrorTests;
+
+  print_test_start (thisTest, func_name, TEST_MAXERROR);
 
   if (max_error == 0.0 || (max_error <= prev_max_error && !ignore_max_ulp))
     {
@@ -392,14 +425,19 @@ print_max_error (const char *func_name)
       printf (" accepted: %s ulp\n", pmestr);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_MAXERROR);
+
+  print_test_end (thisTest, func_name, TEST_MAXERROR);
 }
 
-
+/* This is a builtin test of overall max error.  */
 void
-print_complex_max_error (const char *func_name)
+check_complex_max_error (const char *func_name)
 {
   int real_ok = 0, imag_ok = 0, ok;
+  int thisTest = noMaxErrorTests;
+
+  print_test_start (thisTest, func_name, TEST_MAXERROR);
 
   if (real_max_error == 0
       || (real_max_error <= prev_real_max_error && !ignore_max_ulp))
@@ -436,7 +474,8 @@ print_complex_max_error (const char *func_name)
       printf (" accepted: %s ulp\n", pimestr);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_MAXERROR);
+  print_test_end (thisTest, func_name, TEST_MAXERROR);
 }
 
 
@@ -477,12 +516,13 @@ test_single_exception (const char *test_name,
       else
 	{
 	  if (print_screen (1))
-	    printf ("%s: Exception \"%s\" not set\n", test_name,
+	    printf ("Pass: %s: Exception \"%s\" not set\n", test_name,
 		    flag_name);
 	}
     }
   if (!ok)
     ++noErrors;
+  ++noExcTests;
 }
 #endif
 
@@ -494,23 +534,32 @@ test_exceptions (const char *test_name, int exception)
 {
   if (flag_test_exceptions && EXCEPTION_TESTS (FLOAT))
     {
-      ++noExcTests;
+      int ran = 0;
 #ifdef FE_DIVBYZERO
       if ((exception & DIVIDE_BY_ZERO_EXCEPTION_OK) == 0)
-	test_single_exception (test_name, exception,
+	{
-			       DIVIDE_BY_ZERO_EXCEPTION, FE_DIVBYZERO,
+	  test_single_exception (test_name, exception,
-			       "Divide by zero");
+			         DIVIDE_BY_ZERO_EXCEPTION, FE_DIVBYZERO,
+			         "Divide by zero");
+	  ran = 1;
+	}
 #endif
 #ifdef FE_INVALID
       if ((exception & INVALID_EXCEPTION_OK) == 0)
-	test_single_exception (test_name, exception,
+	{
-			       INVALID_EXCEPTION, FE_INVALID,
+	  test_single_exception (test_name, exception,
-			       "Invalid operation");
+			         INVALID_EXCEPTION, FE_INVALID,
+			         "Invalid operation");
+	  ran = 1;
+	}
 #endif
 #ifdef FE_OVERFLOW
       if ((exception & OVERFLOW_EXCEPTION_OK) == 0)
-	test_single_exception (test_name, exception, OVERFLOW_EXCEPTION,
+	{
-			       FE_OVERFLOW, "Overflow");
+	  test_single_exception (test_name, exception, OVERFLOW_EXCEPTION,
+			         FE_OVERFLOW, "Overflow");
+	  ran = 1;
+	}
 #endif
       /* Spurious "underflow" and "inexact" exceptions are always
 	 allowed for IBM long double, in line with the underlying
@@ -519,17 +568,30 @@ test_exceptions (const char *test_name, int exception)
       if ((exception & UNDERFLOW_EXCEPTION_OK) == 0
 	  && !(test_ibm128
 	       && (exception & UNDERFLOW_EXCEPTION) == 0))
-	test_single_exception (test_name, exception, UNDERFLOW_EXCEPTION,
+	{
-			       FE_UNDERFLOW, "Underflow");
+	  test_single_exception (test_name, exception, UNDERFLOW_EXCEPTION,
+			         FE_UNDERFLOW, "Underflow");
+	  ran = 1;
+	}
+
 #endif
 #ifdef FE_INEXACT
       if ((exception & (INEXACT_EXCEPTION | NO_INEXACT_EXCEPTION)) != 0
 	  && !(test_ibm128
 	       && (exception & NO_INEXACT_EXCEPTION) != 0))
-	test_single_exception (test_name, exception, INEXACT_EXCEPTION,
+	{
-			       FE_INEXACT, "Inexact");
+	  test_single_exception (test_name, exception, INEXACT_EXCEPTION,
+			         FE_INEXACT, "Inexact");
+	  ran = 1;
+	}
 #endif
+      assert (ran == 1);
     }
+   else
+     {
+	if (print_screen (1))
+	  printf ("Info: %s: No exceptions tested\n", test_name);
+     }
   feclearexcept (FE_ALL_EXCEPT);
 }
 
@@ -552,6 +614,7 @@ test_single_errno (const char *test_name, int errno_value,
 	printf ("Failure: %s: errno set to %d, expected %d (%s)\n",
 		test_name, errno_value, expected_value, expected_name);
     }
+  ++noErrnoTests;
 }
 
 /* Test whether errno (value ERRNO_VALUE) has been for TEST_NAME set
@@ -561,13 +624,39 @@ test_errno (const char *test_name, int errno_value, int exceptions)
 {
   if (flag_test_errno)
     {
-      ++noErrnoTests;
+      int ran = 0;
+
+      if ((exceptions & (ERRNO_UNCHANGED|ERRNO_EDOM|ERRNO_ERANGE)) == 0)
+	{
+	  if (print_screen (1))
+	    printf ("Info: %s: The value of errno was not tested\n",
+		    test_name);
+	  return;
+	}
+
+
       if (exceptions & ERRNO_UNCHANGED)
-	test_single_errno (test_name, errno_value, 0, "unchanged");
+	{
+	  test_single_errno (test_name, errno_value, 0, "unchanged");
+	  ran = 1;
+	}
       if (exceptions & ERRNO_EDOM)
-	test_single_errno (test_name, errno_value, EDOM, "EDOM");
+	{
+	  test_single_errno (test_name, errno_value, EDOM, "EDOM");
+	  ran = 1;
+	}
       if (exceptions & ERRNO_ERANGE)
-	test_single_errno (test_name, errno_value, ERANGE, "ERANGE");
+	{
+	  test_single_errno (test_name, errno_value, ERANGE, "ERANGE");
+	  ran = 1;
+	}
+
+      assert (ran == 1);
+    }
+  else
+    {
+	if (print_screen (1))
+	  printf ("Info: %s: No errno tests\n", test_name);
     }
 }
 
@@ -619,6 +708,9 @@ check_float_internal (const char *test_name, FLOAT computed, FLOAT expected,
   FLOAT diff = 0;
   FLOAT ulps = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
@@ -716,12 +808,13 @@ check_float_internal (const char *test_name, FLOAT computed, FLOAT expected,
 	  printf (" max.ulp   : %s\n", mustrn);
 	}
     }
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
 
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -776,12 +869,14 @@ check_int (const char *test_name, int computed, int expected,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -795,11 +890,12 @@ check_int (const char *test_name, int computed, int expected,
       printf (" should be:  %d\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -810,12 +906,14 @@ check_long (const char *test_name, long int computed, long int expected,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -829,11 +927,12 @@ check_long (const char *test_name, long int computed, long int expected,
       printf (" should be:  %ld\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -844,12 +943,14 @@ check_bool (const char *test_name, int computed, int expected,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if ((computed == 0) == (expected == 0))
     ok = 1;
 
@@ -863,11 +964,12 @@ check_bool (const char *test_name, int computed, int expected,
       printf (" should be:  %d\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -879,12 +981,14 @@ check_longlong (const char *test_name, long long int computed,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -898,11 +1002,12 @@ check_longlong (const char *test_name, long long int computed,
       printf (" should be:  %lld\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -913,12 +1018,14 @@ check_intmax_t (const char *test_name, intmax_t computed,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -932,11 +1039,12 @@ check_intmax_t (const char *test_name, intmax_t computed,
       printf (" should be:  %jd\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 
@@ -947,12 +1055,14 @@ check_uintmax_t (const char *test_name, uintmax_t computed,
 {
   int ok = 0;
   int errno_value = errno;
+  int thisTest = noTests;
+
+  print_test_start (thisTest, test_name, TEST_INPUT);
 
   test_exceptions (test_name, exceptions);
   test_errno (test_name, errno_value, exceptions);
   if (exceptions & IGNORE_RESULT)
     goto out;
-  noTests++;
   if (computed == expected)
     ok = 1;
 
@@ -966,11 +1076,12 @@ check_uintmax_t (const char *test_name, uintmax_t computed,
       printf (" should be:  %ju\n", expected);
     }
 
-  update_stats (ok);
+  update_stats (ok, TEST_INPUT);
  out:
   fpstack_test (test_name);
   feclearexcept (FE_ALL_EXCEPT);
   errno = 0;
+  print_test_end (thisTest, test_name, TEST_INPUT);
 }
 
 /* Return whether a test with flags EXCEPTIONS should be run.  */
@@ -1211,9 +1322,11 @@ libm_test_finish (void)
     fclose (ulps_file);
 
   printf ("\nTest suite completed:\n");
-  printf ("  %d test cases plus %d tests for exception flags and\n"
+  printf ("  %d max error test cases,\n", noMaxErrorTests);
-	  "    %d tests for errno executed.\n",
+  printf ("  %d input tests,\n", noTests);
-	  noTests, noExcTests, noErrnoTests);
+  printf ("  - with %d tests for exception flags,\n", noExcTests);
+  printf ("  - with %d tests for errno executed.\n", noErrnoTests);
+
   if (noErrors)
     {
       printf ("  %d errors occurred.\n", noErrors);

math/libm-test-support.h

@@ -170,8 +170,8 @@ extern const char doc[];
 
 int enable_test (int);
 void init_max_error (const char *, int, int);
-void print_max_error (const char *);
+void check_max_error (const char *);
-void print_complex_max_error (const char *);
+void check_complex_max_error (const char *);
 void check_float (const char *, FLOAT, FLOAT, int);
 void check_complex (const char *, CFLOAT, CFLOAT, int);
 void check_int (const char *, int, int, int);