mirror of
https://gitlab.freedesktop.org/mesa/mesa.git
synced 2024-11-23 10:14:13 +08:00
d9857b2745
As for the S3 bucket where the kernel image is stored has been identified and
labeled, the other buckets in use can also be identified and labeled.
cc: mesa-stable
Signed-off-by: Sergi Blanch Torne <sergi.blanch.torne@collabora.com>
Co-developed-by: Guilherme Gallo <guilherme.gallo@collabora.com
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/28979>
(cherry picked from commit 72b3c2e4ba)
352 lines
12 KiB
YAML
352 lines
12 KiB
YAML
# Types of CI pipelines:
|
|
# | pipeline name | context | description |
|
|
# |----------------------|-----------|-------------------------------------------------------------|
|
|
# | merge pipeline | mesa/mesa | pipeline running for an MR; if it passes the MR gets merged |
|
|
# | pre-merge pipeline | mesa/mesa | same as above, except its status doesn't affect the MR |
|
|
# | post-merge pipeline | mesa/mesa | pipeline immediately after merging |
|
|
# | fork pipeline | fork | pipeline running in a user fork |
|
|
# | scheduled pipeline | mesa/mesa | nightly pipelines, running every morning at 4am UTC |
|
|
# | direct-push pipeline | mesa/mesa | when commits are pushed directly to mesa/mesa, bypassing Marge and its gating pipeline |
|
|
#
|
|
# Note that the release branches maintained by the release manager fall under
|
|
# the "direct push" category.
|
|
#
|
|
# "context" indicates the permissions that the jobs get; notably, any
|
|
# container created in mesa/mesa gets pushed immediately for everyone to use
|
|
# as soon as the image tag change is merged.
|
|
#
|
|
# Merge pipelines contain all jobs that must pass before the MR can be merged.
|
|
# Pre-merge pipelines contain the exact same jobs as merge pipelines.
|
|
# Post-merge pipelines contain *only* the `pages` job that deploys the new
|
|
# version of the website.
|
|
# Fork pipelines contain everything.
|
|
# Scheduled pipelines only contain the container+build jobs, and some extra
|
|
# test jobs (typically "full" variants of pre-merge jobs that only run 1/X
|
|
# test cases), but not a repeat of the merge pipeline jobs.
|
|
# Direct-push pipelines contain the same jobs as merge pipelines.
|
|
|
|
workflow:
|
|
rules:
|
|
# do not duplicate pipelines on merge pipelines
|
|
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
|
|
when: never
|
|
# merge pipeline
|
|
- if: &is-merge-attempt $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
variables:
|
|
KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
|
|
MESA_CI_PERFORMANCE_ENABLED: 1
|
|
VALVE_INFRA_VANGOGH_JOB_PRIORITY: "" # Empty tags are ignored by gitlab
|
|
# post-merge pipeline
|
|
- if: &is-post-merge $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "push"
|
|
# nightly pipeline
|
|
- if: &is-scheduled-pipeline $CI_PIPELINE_SOURCE == "schedule"
|
|
variables:
|
|
KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
|
|
JOB_PRIORITY: 50
|
|
VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
|
|
# pipeline for direct pushes that bypassed the CI
|
|
- if: &is-direct-push $CI_PROJECT_NAMESPACE == "mesa" && $CI_PIPELINE_SOURCE == "push" && $GITLAB_USER_LOGIN != "marge-bot"
|
|
variables:
|
|
KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
|
|
JOB_PRIORITY: 40
|
|
VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
|
|
# pre-merge or fork pipeline
|
|
- if: $FORCE_KERNEL_TAG != null
|
|
variables:
|
|
KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${FORCE_KERNEL_TAG}
|
|
JOB_PRIORITY: 50
|
|
VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
|
|
- if: $FORCE_KERNEL_TAG == null
|
|
variables:
|
|
KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
|
|
JOB_PRIORITY: 50
|
|
VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
|
|
|
|
|
|
variables:
|
|
FDO_UPSTREAM_REPO: mesa/mesa
|
|
MESA_TEMPLATES_COMMIT: &ci-templates-commit d5aa3941aa03c2f716595116354fb81eb8012acb
|
|
CI_PRE_CLONE_SCRIPT: |-
|
|
set -o xtrace
|
|
wget -q -O download-git-cache.sh ${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/download-git-cache.sh
|
|
bash download-git-cache.sh
|
|
rm download-git-cache.sh
|
|
set +o xtrace
|
|
S3_JWT_FILE: /s3_jwt
|
|
S3_HOST: s3.freedesktop.org
|
|
# This bucket is used to fetch the kernel image
|
|
S3_KERNEL_BUCKET: mesa-rootfs
|
|
# Bucket for git cache
|
|
S3_GITCACHE_BUCKET: git-cache
|
|
# Bucket for the pipeline artifacts pushed to S3
|
|
S3_ARTIFACTS_BUCKET: artifacts
|
|
# Buckets for traces
|
|
S3_TRACIE_RESULTS_BUCKET: mesa-tracie-results
|
|
S3_TRACIE_PUBLIC_BUCKET: mesa-tracie-public
|
|
S3_TRACIE_PRIVATE_BUCKET: mesa-tracie-private
|
|
# per-pipeline artifact storage on MinIO
|
|
PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/${S3_ARTIFACTS_BUCKET}/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
|
|
# per-job artifact storage on MinIO
|
|
JOB_ARTIFACTS_BASE: ${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}
|
|
# reference images stored for traces
|
|
PIGLIT_REPLAY_REFERENCE_IMAGES_BASE: "${S3_HOST}/${S3_TRACIE_RESULTS_BUCKET}/$FDO_UPSTREAM_REPO"
|
|
# For individual CI farm status see .ci-farms folder
|
|
# Disable farm with `git mv .ci-farms{,-disabled}/$farm_name`
|
|
# Re-enable farm with `git mv .ci-farms{-disabled,}/$farm_name`
|
|
# NEVER MIX FARM MAINTENANCE WITH ANY OTHER CHANGE IN THE SAME MERGE REQUEST!
|
|
ARTIFACTS_BASE_URL: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts
|
|
# Python scripts for structured logger
|
|
PYTHONPATH: "$PYTHONPATH:$CI_PROJECT_DIR/install"
|
|
# Drop once deqp-runner is upgraded to > 0.18.0
|
|
MESA_VK_ABORT_ON_DEVICE_LOSS: 1
|
|
# Avoid the wall of "Unsupported SPIR-V capability" warnings in CI job log, hiding away useful output
|
|
MESA_SPIRV_LOG_LEVEL: error
|
|
|
|
default:
|
|
id_tokens:
|
|
S3_JWT:
|
|
aud: https://s3.freedesktop.org
|
|
before_script:
|
|
- >
|
|
export SCRIPTS_DIR=$(mktemp -d) &&
|
|
curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 -O --output-dir "${SCRIPTS_DIR}" "${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/setup-test-env.sh" &&
|
|
. ${SCRIPTS_DIR}/setup-test-env.sh &&
|
|
echo -n "${S3_JWT}" > "${S3_JWT_FILE}" &&
|
|
unset CI_JOB_JWT S3_JWT # Unsetting vulnerable env variables
|
|
|
|
after_script:
|
|
# Work around https://gitlab.com/gitlab-org/gitlab/-/issues/20338
|
|
- find -name '*.log' -exec mv {} {}.txt \;
|
|
|
|
- >
|
|
set +x
|
|
|
|
test -e "${S3_JWT_FILE}" &&
|
|
export S3_JWT="$(<${S3_JWT_FILE})" &&
|
|
rm "${S3_JWT_FILE}"
|
|
|
|
# Retry when job fails. Failed jobs can be found in the Mesa CI Daily Reports:
|
|
# https://gitlab.freedesktop.org/mesa/mesa/-/issues/?sort=created_date&state=opened&label_name%5B%5D=CI%20daily
|
|
retry:
|
|
max: 1
|
|
# Ignore runner_unsupported, stale_schedule, archived_failure, or
|
|
# unmet_prerequisites
|
|
when:
|
|
- api_failure
|
|
- runner_system_failure
|
|
- script_failure
|
|
- job_execution_timeout
|
|
- scheduler_failure
|
|
- data_integrity_failure
|
|
- unknown_failure
|
|
|
|
stages:
|
|
- sanity
|
|
- container
|
|
- git-archive
|
|
- build-x86_64
|
|
- build-misc
|
|
- code-validation
|
|
- amd
|
|
- intel
|
|
- nouveau
|
|
- arm
|
|
- broadcom
|
|
- freedreno
|
|
- etnaviv
|
|
- software-renderer
|
|
- layered-backends
|
|
- deploy
|
|
|
|
include:
|
|
- project: 'freedesktop/ci-templates'
|
|
ref: 16bc29078de5e0a067ff84a1a199a3760d3b3811
|
|
file:
|
|
- '/templates/ci-fairy.yml'
|
|
- project: 'freedesktop/ci-templates'
|
|
ref: *ci-templates-commit
|
|
file:
|
|
- '/templates/alpine.yml'
|
|
- '/templates/debian.yml'
|
|
- '/templates/fedora.yml'
|
|
- local: '.gitlab-ci/image-tags.yml'
|
|
- local: '.gitlab-ci/lava/lava-gitlab-ci.yml'
|
|
- local: '.gitlab-ci/container/gitlab-ci.yml'
|
|
- local: '.gitlab-ci/build/gitlab-ci.yml'
|
|
- local: '.gitlab-ci/test/gitlab-ci.yml'
|
|
- local: '.gitlab-ci/farm-rules.yml'
|
|
- local: '.gitlab-ci/test-source-dep.yml'
|
|
- local: 'docs/gitlab-ci.yml'
|
|
- local: 'src/**/ci/gitlab-ci.yml'
|
|
|
|
|
|
# YAML anchors for rule conditions
|
|
# --------------------------------
|
|
.rules-anchors:
|
|
# Pre-merge pipeline
|
|
- &is-pre-merge '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
|
|
|
|
.never-post-merge-rules:
|
|
rules:
|
|
- if: *is-post-merge
|
|
when: never
|
|
|
|
|
|
.container+build-rules:
|
|
rules:
|
|
# Run when re-enabling a disabled farm, but not when disabling it
|
|
- !reference [.disable-farm-mr-rules, rules]
|
|
# Never run immediately after merging, as we just ran everything
|
|
- !reference [.never-post-merge-rules, rules]
|
|
# Build everything in merge pipelines, if any files affecting the pipeline
|
|
# were changed
|
|
- if: *is-merge-attempt
|
|
changes: &all_paths
|
|
- VERSION
|
|
- bin/git_sha1_gen.py
|
|
- bin/install_megadrivers.py
|
|
- bin/symbols-check.py
|
|
# GitLab CI
|
|
- .gitlab-ci.yml
|
|
- .gitlab-ci/**/*
|
|
- .ci-farms/*
|
|
# Meson
|
|
- meson*
|
|
- build-support/**/*
|
|
- subprojects/**/*
|
|
# clang format
|
|
- .clang-format
|
|
- .clang-format-include
|
|
- .clang-format-ignore
|
|
# Source code
|
|
- include/**/*
|
|
- src/**/*
|
|
when: on_success
|
|
# Same as above, but for pre-merge pipelines
|
|
- if: *is-pre-merge
|
|
changes:
|
|
*all_paths
|
|
when: manual
|
|
# Skip everything for pre-merge and merge pipelines which don't change
|
|
# anything in the build
|
|
- if: *is-merge-attempt
|
|
when: never
|
|
- if: *is-pre-merge
|
|
when: never
|
|
# Build everything after someone bypassed the CI
|
|
- if: *is-direct-push
|
|
when: on_success
|
|
# Build everything in scheduled pipelines
|
|
- if: *is-scheduled-pipeline
|
|
when: on_success
|
|
# Allow building everything in fork pipelines, but build nothing unless
|
|
# manually triggered
|
|
- when: manual
|
|
|
|
|
|
.ci-deqp-artifacts:
|
|
artifacts:
|
|
name: "mesa_${CI_JOB_NAME}"
|
|
when: always
|
|
untracked: false
|
|
paths:
|
|
# Watch out! Artifacts are relative to the build dir.
|
|
# https://gitlab.com/gitlab-org/gitlab-ce/commit/8788fb925706cad594adf6917a6c5f6587dd1521
|
|
- artifacts
|
|
- _build/meson-logs/*.txt
|
|
- _build/meson-logs/strace
|
|
|
|
# Git archive
|
|
|
|
make git archive:
|
|
extends:
|
|
- .fdo.ci-fairy
|
|
stage: git-archive
|
|
rules:
|
|
- !reference [.scheduled_pipeline-rules, rules]
|
|
# ensure we are running on packet
|
|
tags:
|
|
- packet.net
|
|
script:
|
|
# Compactify the .git directory
|
|
- git gc --aggressive
|
|
# Download & cache the perfetto subproject as well.
|
|
- rm -rf subprojects/perfetto ; mkdir -p subprojects/perfetto && curl https://android.googlesource.com/platform/external/perfetto/+archive/$(grep 'revision =' subprojects/perfetto.wrap | cut -d ' ' -f3).tar.gz | tar zxf - -C subprojects/perfetto
|
|
# compress the current folder
|
|
- tar -cvzf ../$CI_PROJECT_NAME.tar.gz .
|
|
|
|
- ci-fairy s3cp --token-file "${S3_JWT_FILE}" ../$CI_PROJECT_NAME.tar.gz https://$S3_HOST/git-cache/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/$CI_PROJECT_NAME.tar.gz
|
|
|
|
# Sanity checks of MR settings and commit logs
|
|
sanity:
|
|
extends:
|
|
- .fdo.ci-fairy
|
|
stage: sanity
|
|
rules:
|
|
- if: *is-pre-merge
|
|
when: on_success
|
|
- when: never
|
|
variables:
|
|
GIT_STRATEGY: none
|
|
script:
|
|
# ci-fairy check-commits --junit-xml=check-commits.xml
|
|
- ci-fairy check-merge-request --require-allow-collaboration --junit-xml=check-merge-request.xml
|
|
- |
|
|
set -eu
|
|
image_tags=(
|
|
DEBIAN_BASE_TAG
|
|
DEBIAN_BUILD_TAG
|
|
DEBIAN_X86_64_TEST_ANDROID_TAG
|
|
DEBIAN_X86_64_TEST_GL_TAG
|
|
DEBIAN_X86_64_TEST_VK_TAG
|
|
ALPINE_X86_64_BUILD_TAG
|
|
ALPINE_X86_64_LAVA_SSH_TAG
|
|
FEDORA_X86_64_BUILD_TAG
|
|
KERNEL_ROOTFS_TAG
|
|
KERNEL_TAG
|
|
PKG_REPO_REV
|
|
WINDOWS_X64_MSVC_TAG
|
|
WINDOWS_X64_BUILD_TAG
|
|
WINDOWS_X64_TEST_TAG
|
|
)
|
|
for var in "${image_tags[@]}"
|
|
do
|
|
if [ "$(echo -n "${!var}" | wc -c)" -gt 20 ]
|
|
then
|
|
echo "$var is too long; please make sure it is at most 20 chars."
|
|
exit 1
|
|
fi
|
|
done
|
|
artifacts:
|
|
when: on_failure
|
|
reports:
|
|
junit: check-*.xml
|
|
tags:
|
|
- placeholder-job
|
|
|
|
|
|
mr-label-maker-test:
|
|
extends:
|
|
- .fdo.ci-fairy
|
|
stage: sanity
|
|
rules:
|
|
- !reference [.mr-label-maker-rules, rules]
|
|
variables:
|
|
GIT_STRATEGY: fetch
|
|
timeout: 10m
|
|
script:
|
|
- set -eu
|
|
- python3 -m venv .venv
|
|
- source .venv/bin/activate
|
|
- pip install git+https://gitlab.freedesktop.org/freedesktop/mr-label-maker
|
|
- mr-label-maker --dry-run --mr $CI_MERGE_REQUEST_IID
|
|
|
|
# Jobs that need to pass before spending hardware resources on further testing
|
|
.required-for-hardware-jobs:
|
|
needs:
|
|
- job: clang-format
|
|
optional: true
|
|
- job: rustfmt
|
|
optional: true
|