binutils-gdb/gdb/testsuite/analyze-racy-logs.py

#!/usr/bin/env python3

# Copyright (C) 2016-2024 Free Software Foundation, Inc.
#
# This file is part of GDB.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


# This program is used to analyze the test results (i.e., *.sum files)
# generated by GDB's testsuite, and print the testcases that are found
# to be racy.
#
# Racy testcases are considered as being testcases which can
# intermittently FAIL (or PASS) when run two or more times
# consecutively, i.e., tests whose results are not deterministic.
#
# This program is invoked when the user runs "make check" and
# specifies the RACY_ITER environment variable.

import re
import sys

# The (global) dictionary that stores the associations between a *.sum
# file and its results.  The data inside it will be stored as:
#
# files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... },
#                                     'FAIL' : { 'test5', 'test6' ... },
#                                     ...
#                                   },
#                   { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... },
#                                   ...
#                                   }
#                   }

files_and_tests: dict[str, dict[str, set[str]]] = dict()

# The relatioships between various states of the same tests that
# should be ignored.  For example, if the same test PASSes on a
# testcase run but KFAILs on another, this test should be considered
# racy because a known-failure is...  known.

ignore_relations = {"PASS": "KFAIL"}

# We are interested in lines that start with '.?(PASS|FAIL)'.  In
# other words, we don't process errors (maybe we should).

sum_matcher = re.compile("^(.?(PASS|FAIL)): (.*)$")


def parse_sum_line(line: str, dic: dict[str, set[str]]):
    """Parse a single LINE from a sumfile, and store the results in the
    dictionary referenced by DIC."""
    global sum_matcher

    line = line.rstrip()
    m = re.match(sum_matcher, line)
    if m:
        result = m.group(1)
        test_name = m.group(3)
        # Remove tail parentheses.  These are likely to be '(timeout)'
        # and other extra information that will only confuse us.
        test_name = re.sub(r"(\s+)?\(.*$", "", test_name)
        if result not in dic.keys():
            dic[result] = set()
        if test_name in dic[result]:
            # If the line is already present in the dictionary, then
            # we include a unique identifier in the end of it, in the
            # form or '<<N>>' (where N is a number >= 2).  This is
            # useful because the GDB testsuite is full of non-unique
            # test messages; however, if you process the racy summary
            # file you will also need to perform this same operation
            # in order to identify the racy test.
            i = 2
            while True:
                nname = test_name + " <<" + str(i) + ">>"
                if nname not in dic[result]:
                    break
                i += 1
            test_name = nname
        dic[result].add(test_name)


def read_sum_files(files: list[str]):
    """Read the sumfiles (passed as a list in the FILES variable), and
    process each one, filling the FILES_AND_TESTS global dictionary with
    information about them."""
    global files_and_tests

    for x in files:
        with open(x, "r") as f:
            files_and_tests[x] = dict()
            for line in f.readlines():
                parse_sum_line(line, files_and_tests[x])


def identify_racy_tests():
    """Identify and print the racy tests.  This function basically works
    on sets, and the idea behind it is simple.  It takes all the sets that
    refer to the same result (for example, all the sets that contain PASS
    tests), and compare them.  If a test is present in all PASS sets, then
    it is not racy.  Otherwise, it is.

    This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.),
    and then print a sorted list (without duplicates) of all the tests
    that were found to be racy."""
    global files_and_tests

    # First, construct two dictionaries that will hold one set of
    # testcases for each state (PASS, FAIL, etc.).
    #
    # Each set in NONRACY_TESTS will contain only the non-racy
    # testcases for that state.  A non-racy testcase is a testcase
    # that has the same state in all test runs.
    #
    # Each set in ALL_TESTS will contain all tests, racy or not, for
    # that state.
    nonracy_tests: dict[str, set[str]] = dict()
    all_tests: dict[str, set[str]] = dict()
    for f in files_and_tests:
        for state in files_and_tests[f]:
            try:
                nonracy_tests[state] &= files_and_tests[f][state].copy()
            except KeyError:
                nonracy_tests[state] = files_and_tests[f][state].copy()

            try:
                all_tests[state] |= files_and_tests[f][state].copy()
            except KeyError:
                all_tests[state] = files_and_tests[f][state].copy()

    # Now, we eliminate the tests that are present in states that need
    # to be ignored.  For example, tests both in the PASS and KFAIL
    # states should not be considered racy.
    ignored_tests: set[str] = set()
    for s1, s2 in ignore_relations.items():
        try:
            ignored_tests |= all_tests[s1] & all_tests[s2]
        except:
            continue

    racy_tests: set[str] = set()
    for f in files_and_tests:
        for state in files_and_tests[f]:
            racy_tests |= files_and_tests[f][state] - nonracy_tests[state]

    racy_tests = racy_tests - ignored_tests

    # Print the header.
    print("\t\t=== gdb racy tests ===\n")

    # Print each test.
    for line in sorted(racy_tests):
        print(line)

    # Print the summary.
    print("\n")
    print("\t\t=== gdb Summary ===\n")
    print("# of racy tests:\t\t%d" % len(racy_tests))


if __name__ == "__main__":
    if len(sys.argv) < 3:
        # It only makes sense to invoke this program if you pass two
        # or more files to be analyzed.
        sys.exit("Usage: %s [FILE] [FILE] ..." % sys.argv[0])
    read_sum_files(sys.argv[1:])
    identify_racy_tests()
    exit(0)