mirror of
https://gitlab.freedesktop.org/mesa/mesa.git
synced 2024-11-28 04:34:25 +08:00
7801a17b54
skqp output is verbose, as we are running multiple backends at the same job, normally the trace will surpass the Gitlab UI line limit. This commit wraps every skqp execution in a Gitlab section and removes some `set -xtrace` from skqp-runner.sh for a cleaner output. Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17835>
372 lines
11 KiB
Bash
Executable File
372 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Copyright (C) 2022 Collabora Limited
|
|
# Author: Guilherme Gallo <guilherme.gallo@collabora.com>
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
# copy of this software and associated documentation files (the "Software"),
|
|
# to deal in the Software without restriction, including without limitation
|
|
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
# and/or sell copies of the Software, and to permit persons to whom the
|
|
# Software is furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice (including the next
|
|
# paragraph) shall be included in all copies or substantial portions of the
|
|
# Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
|
|
# Args:
|
|
# $1: section id
|
|
# $2: section header
|
|
gitlab_section_start() {
|
|
echo -e "\e[0Ksection_start:$(date +%s):$1[collapsed=${GL_COLLAPSED:-false}]\r\e[0K\e[32;1m$2\e[0m"
|
|
}
|
|
|
|
# Args:
|
|
# $1: section id
|
|
gitlab_section_end() {
|
|
echo -e "\e[0Ksection_end:$(date +%s):$1\r\e[0K"
|
|
}
|
|
|
|
|
|
# sponge allows piping to files that are being used as input.
|
|
# E.g.: sort file.txt | sponge file.txt
|
|
# In order to avoid installing moreutils just to have the sponge binary, we can
|
|
# use a bash function for it
|
|
# Source https://unix.stackexchange.com/a/561346/310927
|
|
sponge () (
|
|
set +x
|
|
append=false
|
|
|
|
while getopts 'a' opt; do
|
|
case $opt in
|
|
a) append=true ;;
|
|
*) echo error; exit 1
|
|
esac
|
|
done
|
|
shift "$(( OPTIND - 1 ))"
|
|
|
|
outfile=$1
|
|
|
|
tmpfile=$(mktemp "$(dirname "$outfile")/tmp-sponge.XXXXXXXX") &&
|
|
cat >"$tmpfile" &&
|
|
if "$append"; then
|
|
cat "$tmpfile" >>"$outfile"
|
|
else
|
|
if [ -f "$outfile" ]; then
|
|
chmod --reference="$outfile" "$tmpfile"
|
|
fi
|
|
if [ -f "$outfile" ]; then
|
|
mv "$tmpfile" "$outfile"
|
|
elif [ -n "$outfile" ] && [ ! -e "$outfile" ]; then
|
|
cat "$tmpfile" >"$outfile"
|
|
else
|
|
cat "$tmpfile"
|
|
fi
|
|
fi &&
|
|
rm -f "$tmpfile"
|
|
)
|
|
|
|
remove_comments_from_files() (
|
|
INPUT_FILES="$*"
|
|
for INPUT_FILE in ${INPUT_FILES}
|
|
do
|
|
[ -f "${INPUT_FILE}" ] || continue
|
|
sed -i '/#/d' "${INPUT_FILE}"
|
|
sed -i '/^\s*$/d' "${INPUT_FILE}"
|
|
done
|
|
)
|
|
|
|
subtract_test_lists() (
|
|
MINUEND=$1
|
|
sort "${MINUEND}" | sponge "${MINUEND}"
|
|
shift
|
|
for SUBTRAHEND in "$@"
|
|
do
|
|
sort "${SUBTRAHEND}" | sponge "${SUBTRAHEND}"
|
|
join -v 1 "${MINUEND}" "${SUBTRAHEND}" |
|
|
sponge "${MINUEND}"
|
|
done
|
|
)
|
|
|
|
merge_rendertests_files() {
|
|
BASE_FILE=$1
|
|
shift
|
|
FILES="$*"
|
|
# shellcheck disable=SC2086
|
|
cat $FILES "$BASE_FILE" |
|
|
sort --unique --stable --field-separator=, --key=1,1 |
|
|
sponge "$BASE_FILE"
|
|
}
|
|
|
|
assure_files() (
|
|
for CASELIST_FILE in $*
|
|
do
|
|
>&2 echo "Looking for ${CASELIST_FILE}..."
|
|
[ -f ${CASELIST_FILE} ] || (
|
|
>&2 echo "Not found. Creating empty."
|
|
touch ${CASELIST_FILE}
|
|
)
|
|
done
|
|
)
|
|
|
|
# Generate rendertests from scratch, customizing with fails/flakes/crashes files
|
|
generate_rendertests() (
|
|
set -e
|
|
GENERATED_FILE=$(mktemp)
|
|
TESTS_FILE_PREFIX="${SKQP_FILE_PREFIX}-${SKQP_BACKEND}_rendertests"
|
|
FLAKES_FILE="${TESTS_FILE_PREFIX}-flakes.txt"
|
|
FAILS_FILE="${TESTS_FILE_PREFIX}-fails.txt"
|
|
CRASHES_FILE="${TESTS_FILE_PREFIX}-crashes.txt"
|
|
RENDER_TESTS_FILE="${TESTS_FILE_PREFIX}.txt"
|
|
|
|
# Default to an empty known flakes file if it doesn't exist.
|
|
assure_files ${FLAKES_FILE} ${FAILS_FILE} ${CRASHES_FILE}
|
|
|
|
# skqp does not support comments in rendertests.txt file
|
|
remove_comments_from_files "${FLAKES_FILE}" "${FAILS_FILE}" "${CRASHES_FILE}"
|
|
|
|
# create an exhaustive rendertest list
|
|
"${SKQP_BIN_DIR}"/list_gms | sort > "$GENERATED_FILE"
|
|
|
|
# Remove undesirable tests from the list
|
|
subtract_test_lists "${GENERATED_FILE}" "${CRASHES_FILE}" "${FLAKES_FILE}"
|
|
|
|
# Add ",0" to each test to set the expected diff sum to zero
|
|
sed -i 's/$/,0/g' "$GENERATED_FILE"
|
|
|
|
merge_rendertests_files "$GENERATED_FILE" "${FAILS_FILE}"
|
|
|
|
mv "${GENERATED_FILE}" "${RENDER_TESTS_FILE}"
|
|
|
|
echo "${RENDER_TESTS_FILE}"
|
|
)
|
|
|
|
generate_unittests() (
|
|
set -e
|
|
GENERATED_FILE=$(mktemp)
|
|
TESTS_FILE_PREFIX="${SKQP_FILE_PREFIX}_unittests"
|
|
FLAKES_FILE="${TESTS_FILE_PREFIX}-flakes.txt"
|
|
FAILS_FILE="${TESTS_FILE_PREFIX}-fails.txt"
|
|
CRASHES_FILE="${TESTS_FILE_PREFIX}-crashes.txt"
|
|
UNIT_TESTS_FILE="${TESTS_FILE_PREFIX}.txt"
|
|
|
|
# Default to an empty known flakes file if it doesn't exist.
|
|
assure_files ${FLAKES_FILE} ${FAILS_FILE} ${CRASHES_FILE}
|
|
|
|
# Remove unitTest_ prefix
|
|
for UT_FILE in "${FAILS_FILE}" "${CRASHES_FILE}" "${FLAKES_FILE}"; do
|
|
sed -i 's/^unitTest_//g' "${UT_FILE}"
|
|
done
|
|
|
|
# create an exhaustive unittests list
|
|
"${SKQP_BIN_DIR}"/list_gpu_unit_tests > "${GENERATED_FILE}"
|
|
|
|
# Remove undesirable tests from the list
|
|
subtract_test_lists "${GENERATED_FILE}" "${CRASHES_FILE}" "${FLAKES_FILE}" "${FAILS_FILE}"
|
|
|
|
remove_comments_from_files "${GENERATED_FILE}"
|
|
mv "${GENERATED_FILE}" "${UNIT_TESTS_FILE}"
|
|
|
|
echo "${UNIT_TESTS_FILE}"
|
|
)
|
|
|
|
run_all_tests() {
|
|
rm -f "${SKQP_ASSETS_DIR}"/skqp/*.txt
|
|
}
|
|
|
|
copy_tests_files() (
|
|
# Copy either unit test or render test files from a specific driver given by
|
|
# GPU VERSION variable.
|
|
# If there is no test file at the expected location, this function will
|
|
# return error_code 1
|
|
SKQP_BACKEND="${1}"
|
|
SKQP_FILE_PREFIX="${INSTALL}/${GPU_VERSION}-skqp"
|
|
|
|
if echo "${SKQP_BACKEND}" | grep -qE 'vk|gl(es)?'
|
|
then
|
|
echo "Generating rendertests.txt file"
|
|
GENERATED_RENDERTESTS=$(generate_rendertests)
|
|
cp "${GENERATED_RENDERTESTS}" "${SKQP_ASSETS_DIR}"/skqp/rendertests.txt
|
|
mkdir -p "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}"
|
|
cp "${GENERATED_RENDERTESTS}" "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}/generated_rendertests.txt"
|
|
return 0
|
|
fi
|
|
|
|
# The unittests.txt path is hardcoded inside assets directory,
|
|
# that is why it needs to be a special case.
|
|
if echo "${SKQP_BACKEND}" | grep -qE "unitTest"
|
|
then
|
|
echo "Generating unittests.txt file"
|
|
GENERATED_UNITTESTS=$(generate_unittests)
|
|
cp "${GENERATED_UNITTESTS}" "${SKQP_ASSETS_DIR}"/skqp/unittests.txt
|
|
mkdir -p "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}"
|
|
cp "${GENERATED_UNITTESTS}" "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}/generated_unittests.txt"
|
|
fi
|
|
)
|
|
|
|
resolve_tests_files() {
|
|
if [ -n "${RUN_ALL_TESTS}" ]
|
|
then
|
|
run_all_tests
|
|
return
|
|
fi
|
|
|
|
SKQP_BACKEND=${1}
|
|
if ! copy_tests_files "${SKQP_BACKEND}"
|
|
then
|
|
echo "No override test file found for ${SKQP_BACKEND}. Using the default one."
|
|
fi
|
|
}
|
|
|
|
test_vk_backend() {
|
|
if echo "${SKQP_BACKENDS:?}" | grep -qE 'vk'
|
|
then
|
|
if [ -n "$VK_DRIVER" ]; then
|
|
return 0
|
|
fi
|
|
|
|
echo "VK_DRIVER environment variable is missing."
|
|
# shellcheck disable=SC2012
|
|
VK_DRIVERS=$(ls "$INSTALL"/share/vulkan/icd.d/ | cut -f 1 -d '_')
|
|
if [ -n "${VK_DRIVERS}" ]
|
|
then
|
|
echo "Please set VK_DRIVER to the correct driver from the list:"
|
|
echo "${VK_DRIVERS}"
|
|
fi
|
|
echo "No Vulkan tests will be executed, but it was requested in SKQP_BACKENDS variable. Exiting."
|
|
exit 2
|
|
fi
|
|
|
|
# Vulkan environment is not configured, but it was not requested by the job
|
|
return 1
|
|
}
|
|
|
|
setup_backends() {
|
|
if test_vk_backend
|
|
then
|
|
export VK_ICD_FILENAMES="$INSTALL"/share/vulkan/icd.d/"$VK_DRIVER"_icd."${VK_CPU:-$(uname -m)}".json
|
|
fi
|
|
}
|
|
|
|
show_reports() (
|
|
set +xe
|
|
|
|
# Unit tests produce empty HTML reports, guide the user to check the TXT file.
|
|
if echo "${SKQP_BACKENDS}" | grep -qE "unitTest"
|
|
then
|
|
# Remove the empty HTML report to avoid confusion
|
|
rm -f "${SKQP_RESULTS_DIR}"/unitTest/report.html
|
|
|
|
echo "See skqp unit test results at:"
|
|
echo "https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts${SKQP_RESULTS_DIR}/unitTest/unit_tests.txt"
|
|
fi
|
|
|
|
REPORT_FILES=$(mktemp)
|
|
find "${SKQP_RESULTS_DIR}"/**/report.html -type f > "${REPORT_FILES}"
|
|
while read -r REPORT
|
|
do
|
|
# shellcheck disable=SC2001
|
|
BACKEND_NAME=$(echo "${REPORT}" | sed 's@.*/\([^/]*\)/report.html@\1@')
|
|
echo "See skqp ${BACKEND_NAME} render tests report at:"
|
|
echo "https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts${REPORT}"
|
|
done < "${REPORT_FILES}"
|
|
|
|
# If there is no report available, tell the user that something is wrong.
|
|
if [ ! -s "${REPORT_FILES}" ]
|
|
then
|
|
echo "No skqp report available. Probably some fatal error has occured during the skqp execution."
|
|
fi
|
|
)
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $(basename "$0") [-a]
|
|
|
|
Arguments:
|
|
-a: Run all unit tests and render tests, useful when introducing a new driver to skqp.
|
|
EOF
|
|
}
|
|
|
|
parse_args() {
|
|
while getopts ':ah' opt; do
|
|
case "$opt" in
|
|
a)
|
|
echo "Running all skqp tests"
|
|
export RUN_ALL_TESTS=1
|
|
shift
|
|
;;
|
|
|
|
h)
|
|
usage
|
|
exit 0
|
|
;;
|
|
|
|
?)
|
|
echo "Invalid command option."
|
|
usage
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
set -e
|
|
|
|
parse_args "${@}"
|
|
|
|
# Needed so configuration files can contain paths to files in /install
|
|
INSTALL="$CI_PROJECT_DIR"/install
|
|
|
|
if [ -z "$GPU_VERSION" ]; then
|
|
echo 'GPU_VERSION must be set to something like "llvmpipe" or
|
|
"freedreno-a630" (it will serve as a component to find the path for files
|
|
residing in src/**/ci/*.txt)'
|
|
exit 1
|
|
fi
|
|
|
|
LD_LIBRARY_PATH=$INSTALL:$LD_LIBRARY_PATH
|
|
setup_backends
|
|
|
|
SKQP_BIN_DIR=${SKQP_BIN_DIR:-/skqp}
|
|
SKQP_ASSETS_DIR="${SKQP_BIN_DIR}"/assets
|
|
SKQP_RESULTS_DIR="${SKQP_RESULTS_DIR:-${PWD}/results}"
|
|
|
|
mkdir -p "${SKQP_ASSETS_DIR}"/skqp
|
|
|
|
# Show the reports on exit, even when a test crashes
|
|
trap show_reports INT TERM EXIT
|
|
|
|
SKQP_EXITCODE=0
|
|
for SKQP_BACKEND in ${SKQP_BACKENDS}
|
|
do
|
|
resolve_tests_files "${SKQP_BACKEND}"
|
|
SKQP_BACKEND_RESULTS_DIR="${SKQP_RESULTS_DIR}"/"${SKQP_BACKEND}"
|
|
mkdir -p "${SKQP_BACKEND_RESULTS_DIR}"
|
|
BACKEND_EXITCODE=0
|
|
|
|
GL_COLLAPSED=true gitlab_section_start "skqp_${SKQP_BACKEND}" "skqp logs for ${SKQP_BACKEND}"
|
|
"${SKQP_BIN_DIR}"/skqp "${SKQP_ASSETS_DIR}" "${SKQP_BACKEND_RESULTS_DIR}" "${SKQP_BACKEND}_" ||
|
|
BACKEND_EXITCODE=$?
|
|
gitlab_section_end "skqp_${SKQP_BACKEND}"
|
|
|
|
if [ ! $BACKEND_EXITCODE -eq 0 ]
|
|
then
|
|
echo "skqp failed on ${SKQP_BACKEND} tests with exit code: ${BACKEND_EXITCODE}."
|
|
else
|
|
echo "skqp succeeded on ${SKQP_BACKEND}."
|
|
fi
|
|
|
|
# Propagate error codes to leverage the final job result
|
|
SKQP_EXITCODE=$(( SKQP_EXITCODE | BACKEND_EXITCODE ))
|
|
done
|
|
|
|
exit $SKQP_EXITCODE
|