mirrors/openssl
0
0
Fork 0
mirror of https://github.com/openssl/openssl.git synced 2024-11-24 10:34:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Rework test/run_tests.pl to support selective verbosity and TAP copy

Browse Source 
This includes a complete rework of how we use TAP::Harness, by adding
a TAP::Parser subclass that allows additional callbacks to be passed
to perform what we need.  The TAP::Parser callbacks we add are:

    ALL         to print all the TAP output to a file (conditionally)
                to collect all the TAP output to an array (conditionally)
    EOF         to print all the collected TAP output (if there is any)
                if any subtest failed

To get TAP output to file, the environment variable HARNESS_TAP_COPY
must be defined, with a file name as value.  That file will be
overwritten unconditionally.

To get TAP output displayed on failure, the make variable VERBOSE_FAILURE
or VF must be defined with a non-emoty value.

Additionally, the output of test recipe names has been changed to only
display its basename.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/9862)
This commit is contained in:
Richard Levitte 2019-09-11 11:05:11 +02:00
parent f3503cb0f6
commit e3d9a6b5f0
4 changed files with 176 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGES
View File

@ -9,6 +9,15 @@
Changes between 1.1.1 and 3.0.0 [xx XXX xxxx]
*) Extended testing to be verbose for failing tests only. The make variables
VERBOSE_FAILURE or VF can be used to enable this:
$ make VF=1 test # Unix
$ mms /macro=(VF=1) test ! OpenVMS
$ nmake VF=1 test # Windows
[Richard Levitte]
*) For built-in EC curves, ensure an EC_GROUP built from the curve name is
used even when parsing explicit parameters, when loading a serialized key
or calling `EC_GROUP_new_from_ecpkparameters()`/

1
Configurations/descrip.mms.tmpl
View File

@ -114,6 +114,7 @@ BLDDIR={- $config{builddir} -}
# Allow both V and VERBOSE to indicate verbosity. This only applies
# to testing.
VERBOSE=$(V)
VERBOSE_FAILURE=$(VF)
VERSION={- "$config{full_version}" -}
MAJOR={- $config{major} -}

14
INSTALL
View File

@ -911,11 +911,17 @@
malfunction with Perl). You may want increased verbosity, that
can be accomplished like this:
$ make VERBOSE=1 test # Unix
Verbosity on failure only (make macro VERBOSE_FAILURE or VF):
$ mms /macro=(VERBOSE=1) test ! OpenVMS
$ make VF=1 test # Unix
$ mms /macro=(VF=1) test ! OpenVMS
$ nmake VF=1 test # Windows
$ nmake VERBOSE=1 test # Windows
Full verbosity (make macro VERBOSE or V):
$ make V=1 test # Unix
$ mms /macro=(V=1) test ! OpenVMS
$ nmake V=1 test # Windows
If you want to run just one or a few specific tests, you can use
the make variable TESTS to specify them, like this:
@ -926,7 +932,7 @@
And of course, you can combine (Unix example shown):
$ make VERBOSE=1 TESTS='test_rsa test_dsa' test
$ make VF=1 TESTS='test_rsa test_dsa' test
You can find the list of available tests like this:

213
test/run_tests.pl
View File

@ -10,8 +10,10 @@ use strict;
use warnings;
# Recognise VERBOSE and V which is common on other projects.
# Additionally, also recognise VERBOSE_FAILURE and VF.
BEGIN {
$ENV{HARNESS_VERBOSE} = "yes" if $ENV{VERBOSE} || $ENV{V};
$ENV{HARNESS_VERBOSE_FAILURE} = "yes" if $ENV{VERBOSE_FAILURE} || $ENV{VF};
}
use File::Spec::Functions qw/catdir catfile curdir abs2rel rel2abs/;
@ -20,9 +22,6 @@ use FindBin;
use lib "$FindBin::Bin/../util/perl";
use OpenSSL::Glob;
my $TAP_Harness = eval { require TAP::Harness } ? "TAP::Harness"
: "OpenSSL::TAP::Harness";
my $srctop = $ENV{SRCTOP} || $ENV{TOP};
my $bldtop = $ENV{BLDTOP} || $ENV{TOP};
my $recipesdir = catdir($srctop, "test", "recipes");
@ -31,12 +30,25 @@ my $libdir = rel2abs(catdir($srctop, "util", "perl"));
$ENV{OPENSSL_CONF} = catdir($srctop, "apps", "openssl.cnf");
my %tapargs =
( verbosity => $ENV{VERBOSE} || $ENV{V} || $ENV{HARNESS_VERBOSE} ? 1 : 0,
lib => [ $libdir ],
switches => '-w',
merge => 1
( verbosity => $ENV{HARNESS_VERBOSE} ? 1 : 0,
lib => [ $libdir ],
switches => '-w',
merge => 1,
);
# Additional OpenSSL special TAP arguments. Because we can't pass them via
# TAP::Harness->new(), they will be accessed directly, see the
# TAP::Parser::OpenSSL implementation further down
my %openssl_args = ();
$openssl_args{'failure_verbosity'} =
$ENV{HARNESS_VERBOSE_FAILURE} && $tapargs{verbosity} < 1 ? 1 : 0;
my $outfilename = $ENV{HARNESS_TAP_COPY};
open $openssl_args{'tap_copy'}, ">$outfilename"
or die "Trying to create $outfilename: $!\n"
if defined $outfilename;
my @alltests = find_matching_tests("*");
my %tests = ();
@ -52,7 +64,7 @@ foreach my $arg (@ARGV ? @ARGV : ('alltests')) {
if ($arg eq 'alltests') {
warn "'alltests' encountered, ignoring everything before that...\n"
unless $initial_arg;
%tests = map { $_ => 1 } @alltests;
%tests = map { $_ => basename($_) } @alltests;
} elsif ($arg =~ m/^(-?)(.*)/) {
my $sign = $1;
my $test = $2;
@ -60,7 +72,7 @@ foreach my $arg (@ARGV ? @ARGV : ('alltests')) {
# If '-foo' is the first arg, it's short for 'alltests -foo'
if ($sign eq '-' && $initial_arg) {
%tests = map { $_ => 1 } @alltests;
%tests = map { $_ => basename($_) } @alltests;
}
if (scalar @matches == 0) {
@ -72,7 +84,7 @@ foreach my $arg (@ARGV ? @ARGV : ('alltests')) {
if ($sign eq '-') {
delete $tests{$test};
} else {
$tests{$test} = 1;
$tests{$test} = basename($test);
}
}
}
@ -83,9 +95,140 @@ foreach my $arg (@ARGV ? @ARGV : ('alltests')) {
$initial_arg = 0;
}
my $harness = $TAP_Harness->new(\%tapargs);
my $ret = $harness->runtests(map { abs2rel($_, rel2abs(curdir())); }
sort keys %tests);
sub find_matching_tests {
my ($glob) = @_;
if ($glob =~ m|^[\d\[\]\?\-]+$|) {
return glob(catfile($recipesdir,"$glob-*.t"));
}
return glob(catfile($recipesdir,"*-$glob.t"));
}
# The following is quite a bit of hackery to adapt to both TAP::Harness
# and Test::Harness, depending on what's available.
# The TAP::Harness hack allows support for HARNESS_VERBOSE_FAILURE and
# HARNESS_TAP_COPY, while the Test::Harness hack can't, because the pre
# TAP::Harness Test::Harness simply doesn't have support for this sort of
# thing.
#
# We use eval to avoid undue interruption if TAP::Harness isn't present.
my $package;
my $eres;
$eres = eval {
package TAP::Parser::OpenSSL;
use parent 'TAP::Parser';
sub new {
my $class = shift;
my %opts = %{ shift() };
# We rely heavily on perl closures to make failure verbosity work
# We need to do so, because there's no way to safely pass extra
# objects down all the way to the TAP::Parser::Result object
my @failure_output = ();
my %callbacks = ();
if ($openssl_args{failure_verbosity}
|| defined $openssl_args{tap_copy}) {
$callbacks{ALL} = sub {
my $self = shift;
my $fh = $openssl_args{tap_copy};
print $fh $self->as_string, "\n"
if defined $fh;
push @failure_output, $self->as_string
if $openssl_args{failure_verbosity} > 0;
};
}
if ($openssl_args{failure_verbosity} > 0) {
$callbacks{EOF} = sub {
my $self = shift;
# We know we are a TAP::Parser::Aggregator object
if (scalar $self->failed > 0 && @failure_output) {
# We add an extra empty line, because in the case of a
# progress counter, we're still at the end of that progress
# line.
print $_, "\n" foreach (("", @failure_output));
}
};
}
if (keys %callbacks) {
# If %opts already has a callbacks element, the order here
# ensures we do not override it
%opts = ( callbacks => { %callbacks }, %opts );
}
return $class->SUPER::new({ %opts });
}
package TAP::Harness::OpenSSL;
use parent 'TAP::Harness';
package main;
$tapargs{parser_class} = "TAP::Parser::OpenSSL";
$package = 'TAP::Harness::OpenSSL';
};
unless (defined $eres) {
$eres = eval {
# Fake TAP::Harness in case it's not loaded
package TAP::Harness::fake;
use parent 'Test::Harness';
sub new {
my $class = shift;
my %args = %{ shift() };
return bless { %args }, $class;
}
sub runtests {
my $self = shift;
# Pre TAP::Harness Test::Harness doesn't support [ filename, name ]
# elements, so convert such elements to just be the filename
my @args = map { ref($_) eq 'ARRAY' ? $_->[0] : $_ } @_;
my @switches = ();
if ($self->{switches}) {
push @switches, $self->{switches};
}
if ($self->{lib}) {
foreach (@{$self->{lib}}) {
my $l = $_;
# It seems that $switches is getting interpreted with 'eval'
# or something like that, and that we need to take care of
# backslashes or they will disappear along the way.
$l =~ s|\\|\\\\|g if $^O eq "MSWin32";
push @switches, "-I$l";
}
}
$Test::Harness::switches = join(' ', @switches);
Test::Harness::runtests(@args);
}
package main;
$package = 'TAP::Harness::fake';
};
}
unless (defined $eres) {
print $@,"\n" if $@;
print $!,"\n" if $!;
exit 127;
}
my $harness = $package->new(\%tapargs);
my $ret =
$harness->runtests(map { [ abs2rel($_, rel2abs(curdir())), $tests{$_} ] }
sort keys %tests);
# $ret->has_errors may be any number, not just 0 or 1. On VMS, numbers
# from 2 and on are used as is as VMS statuses, which has severity encoded
@ -97,47 +240,3 @@ exit !!$ret->has_errors if (ref($ret) eq "TAP::Parser::Aggregator");
# If this isn't a TAP::Parser::Aggregator, it's the pre-TAP test harness,
# which simply dies at the end if any test failed, so we don't need to bother
# with any exit code in that case.
sub find_matching_tests {
my ($glob) = @_;
if ($glob =~ m|^[\d\[\]\?\-]+$|) {
return glob(catfile($recipesdir,"$glob-*.t"));
}
return glob(catfile($recipesdir,"*-$glob.t"));
}
# Fake TAP::Harness in case it's not loaded
use Test::Harness;
package OpenSSL::TAP::Harness;
sub new {
my $class = shift;
my %args = %{ shift() };
return bless { %args }, $class;
}
sub runtests {
my $self = shift;
my @switches = ();
if ($self->{switches}) {
push @switches, $self->{switches};
}
if ($self->{lib}) {
foreach (@{$self->{lib}}) {
my $l = $_;
# It seems that $switches is getting interpreted with 'eval' or
# something like that, and that we need to take care of backslashes
# or they will disappear along the way.
$l =~ s|\\|\\\\|g if $^O eq "MSWin32";
push @switches, "-I$l";
}
}
$Test::Harness::switches = join(' ', @switches);
Test::Harness::runtests(@_);
}