mesa/.gitlab-ci/bare-metal/fastboot_run.py

#!/usr/bin/env python3
#
# Copyright © 2020 Google LLC
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice (including the next
# paragraph) shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

import argparse
import subprocess
import re
from serial_buffer import SerialBuffer
import sys
import threading


class FastbootRun:
    def __init__(self, args, test_timeout):
        self.powerup = args.powerup
        self.ser = SerialBuffer(
            args.dev, "results/serial-output.txt", "R SERIAL> ")
        self.fastboot = "fastboot boot -s {ser} artifacts/fastboot.img".format(
            ser=args.fbserial)
        self.test_timeout = test_timeout

    def close(self):
        self.ser.close()

    def print_error(self, message):
        RED = '\033[0;31m'
        NO_COLOR = '\033[0m'
        print(RED + message + NO_COLOR)

    def logged_system(self, cmd, timeout=60):
        print("Running '{}'".format(cmd))
        try:
            return subprocess.call(cmd, shell=True, timeout=timeout)
        except subprocess.TimeoutExpired:
            self.print_error("timeout, restarting run...")
            return 2

    def run(self):
        if ret := self.logged_system(self.powerup):
            return ret

        fastboot_ready = False
        for line in self.ser.lines(timeout=2 * 60, phase="bootloader"):
            if re.search("fastboot: processing commands", line) or \
                    re.search("Listening for fastboot command on", line):
                fastboot_ready = True
                break

            if re.search("data abort", line):
                self.print_error(
                    "Detected crash during boot, restarting run...")
                return 2

        if not fastboot_ready:
            self.print_error(
                "Failed to get to fastboot prompt, restarting run...")
            return 2

        if ret := self.logged_system(self.fastboot):
            return ret

        print_more_lines = -1
        for line in self.ser.lines(timeout=self.test_timeout, phase="test"):
            if print_more_lines == 0:
                return 2
            if print_more_lines > 0:
                print_more_lines -= 1

            if re.search("---. end Kernel panic", line):
                return 1

            # The db820c boards intermittently reboot.  Just restart the run
            # when if we see a reboot after we got past fastboot.
            if re.search("PON REASON", line):
                self.print_error(
                    "Detected spontaneous reboot, restarting run...")
                return 2

            # db820c sometimes wedges around iommu fault recovery
            if re.search("watchdog: BUG: soft lockup - CPU.* stuck", line):
                self.print_error(
                    "Detected kernel soft lockup, restarting run...")
                return 2

            # If the network device dies, it's probably not graphics's fault, just try again.
            if re.search("NETDEV WATCHDOG", line):
                self.print_error(
                    "Detected network device failure, restarting run...")
                return 2

            # A3xx recovery doesn't quite work. Sometimes the GPU will get
            # wedged and recovery will fail (because power can't be reset?)
            # This assumes that the jobs are sufficiently well-tested that GPU
            # hangs aren't always triggered, so just try again. But print some
            # more lines first so that we get better information on the cause
            # of the hang. Once a hang happens, it's pretty chatty.
            if "[drm:adreno_recover] *ERROR* gpu hw init failed: -22" in line:
                self.print_error(
                    "Detected GPU hang, restarting run...")
                if print_more_lines == -1:
                    print_more_lines = 30

            result = re.search("hwci: mesa: (\S*)", line)
            if result:
                if result.group(1) == "pass":
                    return 0
                else:
                    return 1

        self.print_error(
            "Reached the end of the CPU serial log without finding a result, restarting run...")
        return 2


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--dev', type=str, help='Serial device (otherwise reading from serial-output.txt)')
    parser.add_argument('--powerup', type=str,
                        help='shell command for rebooting', required=True)
    parser.add_argument('--powerdown', type=str,
                        help='shell command for powering off', required=True)
    parser.add_argument('--fbserial', type=str,
                        help='fastboot serial number of the board', required=True)
    parser.add_argument('--test-timeout', type=int,
                        help='Test phase timeout (minutes)', required=True)
    args = parser.parse_args()

    fastboot = FastbootRun(args, args.test_timeout * 60)

    while True:
        retval = fastboot.run()
        fastboot.close()
        if retval != 2:
            break

        fastboot = FastbootRun(args, args.test_timeout * 60)

    fastboot.logged_system(args.powerdown)

    sys.exit(retval)


if __name__ == '__main__':
    main()