gitlab: convert build/container jobs to .base_job_template

This converts the main build and container jobs to use the base job rules, defining the following new variables - QEMU_JOB_SKIPPED - jobs that are known to be currently broken and should not be run. Can still be manually launched if desired. - QEMU_JOB_AVOCADO - jobs that run the Avocado integration test harness. - QEMU_JOB_PUBLISH - jobs that publish content after the branch is merged upstream As build-tools-and-docs runs on master we declare the requirement of building amd64-debian-container optional as it should already exits once we merge. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> Message-Id: <20220526110705.59952-5-berrange@redhat.com> [AJB: fix upstream typo, mention optional container req] Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Reviewed-by: Thomas Huth <thuth@redhat.com> Message-Id: <20220527153603.887929-32-alex.bennee@linaro.org>
2024-11-23 19:03:38 +08:00 · 2022-05-27 16:36:01 +01:00 · 2022-05-27 16:36:01 +01:00 · e312d1fdbb
commit e312d1fdbb
parent 16fee101d9
8 changed files with 65 additions and 31 deletions
--- a/.gitlab-ci.d/base.yml
+++ b/.gitlab-ci.d/base.yml
@ -16,10 +16,22 @@
    - if: '$QEMU_JOB_CIRRUS && ($CIRRUS_GITHUB_REPO == "" || $CIRRUS_API_TOKEN == "")'
      when: never

+    # Publishing jobs should only run on the default branch in upstream
+    - if: '$QEMU_JOB_PUBLISH == "1" && $CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
+      when: never
+
+    # Non-publishing jobs should only run on staging branches in upstream
+    - if: '$QEMU_JOB_PUBLISH != "1" && $CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH !~ /staging/'
+      when: never
+
    # Jobs only intended for forks should always be skipped on upstream
    - if: '$QEMU_JOB_ONLY_FORKS == "1" && $CI_PROJECT_NAMESPACE == "qemu-project"'
      when: never

+    # Avocado jobs don't run in forks unless $QEMU_CI_AVOCADO_TESTING is set
+    - if: '$QEMU_JOB_AVOCADO && $QEMU_CI_AVOCADO_TESTING != "1" && $CI_PROJECT_NAMESPACE != "qemu-project"'
+      when: never
+

    #############################################################
    # Stage 2: fine tune execution of jobs in specific scenarios
@ -31,6 +43,16 @@
      when: manual
      allow_failure: true

+    # Skipped jobs should not be run unless manually triggered
+    - if: '$QEMU_JOB_SKIPPED'
+      when: manual
+      allow_failure: true
+
+    # Avocado jobs can be manually start in forks if $QEMU_CI_AVOCADO_TESTING is unset
+    - if: '$QEMU_JOB_AVOCADO && $CI_PROJECT_NAMESPACE != "qemu-project"'
+      when: manual
+      allow_failure: true
+

    #############################################################
    # Stage 3: catch all logic applying to any job not matching
--- a/.gitlab-ci.d/buildtest-template.yml
+++ b/.gitlab-ci.d/buildtest-template.yml
@ -1,4 +1,5 @@
 .native_build_job_template:
+  extends: .base_job_template
  stage: build
  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
  before_script:
@ -27,6 +28,7 @@
      fi

 .common_test_job_template:
+  extends: .base_job_template
  stage: test
  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
  script:
@ -77,15 +79,5 @@
  after_script:
    - cd build
    - du -chs ${CI_PROJECT_DIR}/avocado-cache
-  rules:
-    # Only run these jobs if running on the mainstream namespace,
-    # or if the user set the QEMU_CI_AVOCADO_TESTING variable (either
-    # in its namespace setting or via git-push option, see documentation
-    # in /.gitlab-ci.yml of this repository).
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project"'
-      when: on_success
-    - if: '$QEMU_CI_AVOCADO_TESTING'
-      when: on_success
-    # Otherwise, set to manual (the jobs are created but not run).
-    - when: manual
-      allow_failure: true
+  variables:
+    QEMU_JOB_AVOCADO: 1
--- a/.gitlab-ci.d/buildtest.yml
+++ b/.gitlab-ci.d/buildtest.yml
@ -360,12 +360,11 @@ build-cfi-aarch64:
    expire_in: 2 days
    paths:
      - build
-  rules:
+  variables:
    # FIXME: This job is often failing, likely due to out-of-memory problems in
    # the constrained containers of the shared runners. Thus this is marked as
-    # manual until the situation has been solved.
-    - when: manual
-      allow_failure: true
+    # skipped until the situation has been solved.
+    QEMU_JOB_SKIPPED: 1

 check-cfi-aarch64:
  extends: .native_test_job_template
@ -402,12 +401,11 @@ build-cfi-ppc64-s390x:
    expire_in: 2 days
    paths:
      - build
-  rules:
+  variables:
    # FIXME: This job is often failing, likely due to out-of-memory problems in
    # the constrained containers of the shared runners. Thus this is marked as
-    # manual until the situation has been solved.
-    - when: manual
-      allow_failure: true
+    # skipped until the situation has been solved.
+    QEMU_JOB_SKIPPED: 1

 check-cfi-ppc64-s390x:
  extends: .native_test_job_template
@ -579,6 +577,7 @@ build-without-default-features:
    MAKE_CHECK_ARGS: check-unit check-qtest SPEED=slow

 build-libvhost-user:
+  extends: .base_job_template
  stage: build
  image: $CI_REGISTRY_IMAGE/qemu/fedora:latest
  needs:
@ -595,10 +594,13 @@ build-tools-and-docs-debian:
  extends: .native_build_job_template
  needs:
    job: amd64-debian-container
+    # when running on 'master' we use pre-existing container
+    optional: true
  variables:
    IMAGE: debian-amd64
    MAKE_CHECK_ARGS: check-unit check-softfloat ctags TAGS cscope
    CONFIGURE_ARGS: --disable-system --disable-user --enable-docs --enable-tools
+    QEMU_JOB_PUBLISH: 1
  artifacts:
    expire_in: 2 days
    paths:
@ -618,6 +620,7 @@ build-tools-and-docs-debian:
 # that users can see the results of their commits, regardless
 # of what topic branch they're currently using
 pages:
+  extends: .base_job_template
  image: $CI_REGISTRY_IMAGE/qemu/debian-amd64:latest
  stage: test
  needs:
@ -635,10 +638,5 @@ pages:
  artifacts:
    paths:
      - public
-  rules:
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
-      when: on_success
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project"'
-      when: never
-    - if: '$CI_PROJECT_NAMESPACE != "qemu-project"'
-      when: on_success
+  variables:
+    QEMU_JOB_PUBLISH: 1
--- a/.gitlab-ci.d/container-cross.yml
+++ b/.gitlab-ci.d/container-cross.yml
@ -40,15 +40,13 @@ armhf-debian-cross-container:
 # We never want to build hexagon in the CI system and by default we
 # always want to refer to the master registry where it lives.
 hexagon-cross-container:
+  extends: .base_job_template
  image: docker:stable
  stage: containers
-  rules:
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project"'
-      when: never
-    - when: always
  variables:
    NAME: debian-hexagon-cross
    GIT_DEPTH: 1
+    QEMU_JOB_ONLY_FORKS: 1
  services:
    - docker:dind
  before_script:
--- a/.gitlab-ci.d/container-template.yml
+++ b/.gitlab-ci.d/container-template.yml
@ -1,4 +1,5 @@
 .container_job_template:
+  extends: .base_job_template
  image: docker:stable
  stage: containers
  services:
--- a/.gitlab-ci.d/crossbuild-template.yml
+++ b/.gitlab-ci.d/crossbuild-template.yml
@ -1,4 +1,5 @@
 .cross_system_build_job:
+  extends: .base_job_template
  stage: build
  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
  timeout: 80m
@ -24,6 +25,7 @@
 # KVM), and set extra options (such disabling other accelerators) via the
 # $EXTRA_CONFIGURE_OPTS variable.
 .cross_accel_build_job:
+  extends: .base_job_template
  stage: build
  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
  timeout: 30m
@ -36,6 +38,7 @@
    - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS

 .cross_user_build_job:
+  extends: .base_job_template
  stage: build
  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
  script:
--- a/.gitlab-ci.d/windows.yml
+++ b/.gitlab-ci.d/windows.yml
@ -1,4 +1,5 @@
 .shared_msys2_builder:
+  extends: .base_job_template
  tags:
  - shared-windows
  - windows
--- a/docs/devel/ci-jobs.rst.inc
+++ b/docs/devel/ci-jobs.rst.inc
@ -73,6 +73,25 @@ The job results are only of interest to contributors prior to
 submitting code. They are not required as part of the gating
 CI pipeline.

+QEMU_JOB_SKIPPED
+~~~~~~~~~~~~~~~~
+
+The job is not reliably successsful in general, so is not
+currently suitable to be run by default. Ideally this should
+be a temporary marker until the problems can be addressed, or
+the job permanently removed.
+
+QEMU_JOB_PUBLISH
+~~~~~~~~~~~~~~~~
+
+The job is for publishing content after a branch has been
+merged into the upstream default branch.
+
+QEMU_JOB_AVOCADO
+~~~~~~~~~~~~~~~~
+
+The job runs the Avocado integration test suite
+
 Contributor controlled runtime variables
 ----------------------------------------