From 8132d38120ccad47977f1082c4651e8539507cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20P=2E=20Berrang=C3=A9?= Date: Thu, 25 Apr 2024 14:42:32 +0100 Subject: [PATCH] ci: refresh with latest lcitool manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This brings in a fix to the job rules which solves a problem with jobs getting skipped in merge requests in some scenarios. It also changes the way Cirrus CI vars are set, which involves a weak to the way $PATH is set in build.yml. Reviewed-by: Michal Privoznik Signed-off-by: Daniel P. Berrangé --- ci/buildenv/alpine-319.sh | 2 +- ci/buildenv/alpine-edge.sh | 2 +- ci/buildenv/centos-stream-9.sh | 1 + ci/cirrus/build.yml | 2 +- ci/containers/alpine-319.Dockerfile | 2 +- ci/containers/alpine-edge.Dockerfile | 2 +- ci/containers/centos-stream-9.Dockerfile | 1 + ci/gitlab/build-templates.yml | 196 ++++++++++------------- 8 files changed, 96 insertions(+), 112 deletions(-) diff --git a/ci/buildenv/alpine-319.sh b/ci/buildenv/alpine-319.sh index c1aedf79b8..43fcb38a46 100644 --- a/ci/buildenv/alpine-319.sh +++ b/ci/buildenv/alpine-319.sh @@ -69,7 +69,7 @@ function install_buildenv() { xen-dev \ yajl-dev rm -f /usr/lib*/python3*/EXTERNALLY-MANAGED - apk list | sort > /packages.txt + apk list --installed | sort > /packages.txt mkdir -p /usr/libexec/ccache-wrappers ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/cc ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/clang diff --git a/ci/buildenv/alpine-edge.sh b/ci/buildenv/alpine-edge.sh index c1aedf79b8..43fcb38a46 100644 --- a/ci/buildenv/alpine-edge.sh +++ b/ci/buildenv/alpine-edge.sh @@ -69,7 +69,7 @@ function install_buildenv() { xen-dev \ yajl-dev rm -f /usr/lib*/python3*/EXTERNALLY-MANAGED - apk list | sort > /packages.txt + apk list --installed | sort > /packages.txt mkdir -p /usr/libexec/ccache-wrappers ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/cc ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/clang diff --git a/ci/buildenv/centos-stream-9.sh b/ci/buildenv/centos-stream-9.sh index d5cfde517f..8dabda22b3 100644 --- a/ci/buildenv/centos-stream-9.sh +++ b/ci/buildenv/centos-stream-9.sh @@ -43,6 +43,7 @@ function install_buildenv() { libblkid-devel \ libcap-ng-devel \ libcurl-devel \ + libiscsi-devel \ libnbd-devel \ libnl3-devel \ libpcap-devel \ diff --git a/ci/cirrus/build.yml b/ci/cirrus/build.yml index 0ae5c2ce64..c0ac05f4d9 100644 --- a/ci/cirrus/build.yml +++ b/ci/cirrus/build.yml @@ -6,7 +6,7 @@ env: CI_COMMIT_REF_NAME: "@CI_COMMIT_REF_NAME@" CI_MERGE_REQUEST_REF_PATH: "@CI_MERGE_REQUEST_REF_PATH@" CI_COMMIT_SHA: "@CI_COMMIT_SHA@" - PATH: "@PATH@" + PATH: "@PATH_EXTRA@:$PATH" PKG_CONFIG_PATH: "@PKG_CONFIG_PATH@" PYTHON: "@PYTHON@" MAKE: "@MAKE@" diff --git a/ci/containers/alpine-319.Dockerfile b/ci/containers/alpine-319.Dockerfile index 1a7a5c4d69..2455184a87 100644 --- a/ci/containers/alpine-319.Dockerfile +++ b/ci/containers/alpine-319.Dockerfile @@ -70,7 +70,7 @@ RUN apk update && \ xen-dev \ yajl-dev && \ rm -f /usr/lib*/python3*/EXTERNALLY-MANAGED && \ - apk list | sort > /packages.txt && \ + apk list --installed | sort > /packages.txt && \ mkdir -p /usr/libexec/ccache-wrappers && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/cc && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/clang && \ diff --git a/ci/containers/alpine-edge.Dockerfile b/ci/containers/alpine-edge.Dockerfile index dfe233a1d5..b28c96692c 100644 --- a/ci/containers/alpine-edge.Dockerfile +++ b/ci/containers/alpine-edge.Dockerfile @@ -70,7 +70,7 @@ RUN apk update && \ xen-dev \ yajl-dev && \ rm -f /usr/lib*/python3*/EXTERNALLY-MANAGED && \ - apk list | sort > /packages.txt && \ + apk list --installed | sort > /packages.txt && \ mkdir -p /usr/libexec/ccache-wrappers && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/cc && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/clang && \ diff --git a/ci/containers/centos-stream-9.Dockerfile b/ci/containers/centos-stream-9.Dockerfile index 84465a8df9..082b18d06f 100644 --- a/ci/containers/centos-stream-9.Dockerfile +++ b/ci/containers/centos-stream-9.Dockerfile @@ -44,6 +44,7 @@ RUN dnf distro-sync -y && \ libblkid-devel \ libcap-ng-devel \ libcurl-devel \ + libiscsi-devel \ libnbd-devel \ libnl3-devel \ libpcap-devel \ diff --git a/ci/gitlab/build-templates.yml b/ci/gitlab/build-templates.yml index 75d9a6f127..b1e41b0783 100644 --- a/ci/gitlab/build-templates.yml +++ b/ci/gitlab/build-templates.yml @@ -37,7 +37,7 @@ variables: IMAGE: $CI_REGISTRY/$RUN_UPSTREAM_NAMESPACE/libvirt/ci-$NAME:latest rules: - ### Rules where we expect to use pre-built container images + ### PUSH events # upstream: pushes to the default branch - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' @@ -53,28 +53,6 @@ - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV' when: on_success - # upstream: other web/api/scheduled pipelines targeting the default branch - - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' - when: manual - allow_failure: true - - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH' - when: on_success - - # upstream+forks: merge requests targeting the default branch, without CI changes - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' - changes: - - ci/gitlab/container-templates.yml - - ci/containers/$NAME.Dockerfile - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' - when: manual - allow_failure: true - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' - when: on_success - - - ### Rules where we need to use the target base container image - # forks: pushes to branches with pipeline requested - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE && $JOB_OPTIONAL' when: manual @@ -86,27 +64,8 @@ variables: IMAGE: $TARGET_BASE_IMAGE - # upstream: other web/api/scheduled pipelines targeting non-default branches - - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' - when: manual - allow_failure: true - variables: - IMAGE: $TARGET_BASE_IMAGE - - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH' - when: on_success - variables: - IMAGE: $TARGET_BASE_IMAGE - # forks: other web/api/scheduled pipelines - - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $JOB_OPTIONAL' - when: manual - allow_failure: true - variables: - IMAGE: $TARGET_BASE_IMAGE - - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/' - when: on_success - variables: - IMAGE: $TARGET_BASE_IMAGE + ### MERGE REQUEST events # upstream+forks: merge requests targeting the default branch, with CI changes - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' @@ -125,6 +84,13 @@ variables: IMAGE: $TARGET_BASE_IMAGE + # upstream+forks: merge requests targeting the default branch + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' + when: manual + allow_failure: true + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' + when: on_success + # upstream+forks: merge requests targeting non-default branches - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' when: manual @@ -136,7 +102,40 @@ variables: IMAGE: $TARGET_BASE_IMAGE - ### Neither prebuilt or local container images + + ### WEB / API / SCHEDULED events + + # upstream: other web/api/scheduled pipelines targeting the default branch + - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' + when: manual + allow_failure: true + - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH' + when: on_success + + # upstream: other web/api/scheduled pipelines targeting non-default branches + - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' + when: manual + allow_failure: true + variables: + IMAGE: $TARGET_BASE_IMAGE + - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH' + when: on_success + variables: + IMAGE: $TARGET_BASE_IMAGE + + # forks: other web/api/scheduled pipelines on any branches + - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $JOB_OPTIONAL' + when: manual + allow_failure: true + variables: + IMAGE: $TARGET_BASE_IMAGE + - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/' + when: on_success + variables: + IMAGE: $TARGET_BASE_IMAGE + + + ### Catch all unhandled events # upstream+forks: that's all folks - when: never @@ -174,7 +173,7 @@ variables: IMAGE: $CI_REGISTRY/$RUN_UPSTREAM_NAMESPACE/libvirt/ci-$NAME-cross-$CROSS:latest rules: - ### Rules where we expect to use pre-built container images + ### PUSH events # upstream: pushes to the default branch - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' @@ -190,28 +189,6 @@ - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE_UPSTREAM_ENV' when: on_success - # upstream: other web/api/scheduled pipelines targeting the default branch - - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' - when: manual - allow_failure: true - - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH' - when: on_success - - # upstream+forks: merge requests targeting the default branch, without CI changes - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' - changes: - - ci/gitlab/container-templates.yml - - ci/containers/$NAME-cross-$CROSS.Dockerfile - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' - when: manual - allow_failure: true - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' - when: on_success - - - ### Rules where we need to use the target base container image - # forks: pushes to branches with pipeline requested - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE == "push" && $RUN_PIPELINE && $JOB_OPTIONAL' when: manual @@ -223,27 +200,8 @@ variables: IMAGE: $TARGET_BASE_IMAGE - # upstream: other web/api/scheduled pipelines targeting non-default branches - - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' - when: manual - allow_failure: true - variables: - IMAGE: $TARGET_BASE_IMAGE - - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH' - when: on_success - variables: - IMAGE: $TARGET_BASE_IMAGE - # forks: other web/api/scheduled pipelines - - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $JOB_OPTIONAL' - when: manual - allow_failure: true - variables: - IMAGE: $TARGET_BASE_IMAGE - - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/' - when: on_success - variables: - IMAGE: $TARGET_BASE_IMAGE + ### MERGE REQUEST events # upstream+forks: merge requests targeting the default branch, with CI changes - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' @@ -262,6 +220,13 @@ variables: IMAGE: $TARGET_BASE_IMAGE + # upstream+forks: merge requests targeting the default branch + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' + when: manual + allow_failure: true + - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH' + when: on_success + # upstream+forks: merge requests targeting non-default branches - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME != $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' when: manual @@ -273,7 +238,40 @@ variables: IMAGE: $TARGET_BASE_IMAGE - ### Neither prebuilt or local container images + + ### WEB / API / SCHEDULED events + + # upstream: other web/api/scheduled pipelines targeting the default branch + - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' + when: manual + allow_failure: true + - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH' + when: on_success + + # upstream: other web/api/scheduled pipelines targeting non-default branches + - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH && $JOB_OPTIONAL' + when: manual + allow_failure: true + variables: + IMAGE: $TARGET_BASE_IMAGE + - if: '$CI_PROJECT_NAMESPACE == $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH' + when: on_success + variables: + IMAGE: $TARGET_BASE_IMAGE + + # forks: other web/api/scheduled pipelines on any branches + - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/ && $JOB_OPTIONAL' + when: manual + allow_failure: true + variables: + IMAGE: $TARGET_BASE_IMAGE + - if: '$CI_PROJECT_NAMESPACE != $RUN_UPSTREAM_NAMESPACE && $CI_PIPELINE_SOURCE =~ /(web|api|schedule)/' + when: on_success + variables: + IMAGE: $TARGET_BASE_IMAGE + + + ### Catch all unhandled events # upstream+forks: that's all folks - when: never @@ -285,26 +283,10 @@ interruptible: true needs: [] script: + - set -o allexport - source ci/cirrus/$NAME.vars - - sed -e "s|[@]CI_REPOSITORY_URL@|$CI_REPOSITORY_URL|g" - -e "s|[@]CI_COMMIT_REF_NAME@|$CI_COMMIT_REF_NAME|g" - -e "s|[@]CI_MERGE_REQUEST_REF_PATH@|$CI_MERGE_REQUEST_REF_PATH|g" - -e "s|[@]CI_COMMIT_SHA@|$CI_COMMIT_SHA|g" - -e "s|[@]CIRRUS_VM_INSTANCE_TYPE@|$CIRRUS_VM_INSTANCE_TYPE|g" - -e "s|[@]CIRRUS_VM_IMAGE_SELECTOR@|$CIRRUS_VM_IMAGE_SELECTOR|g" - -e "s|[@]CIRRUS_VM_IMAGE_NAME@|$CIRRUS_VM_IMAGE_NAME|g" - -e "s|[@]UPDATE_COMMAND@|$UPDATE_COMMAND|g" - -e "s|[@]UPGRADE_COMMAND@|$UPGRADE_COMMAND|g" - -e "s|[@]INSTALL_COMMAND@|$INSTALL_COMMAND|g" - -e "s|[@]PATH@|$PATH_EXTRA${PATH_EXTRA:+:}\$PATH|g" - -e "s|[@]PKG_CONFIG_PATH@|$PKG_CONFIG_PATH|g" - -e "s|[@]PKGS@|$PKGS|g" - -e "s|[@]MAKE@|$MAKE|g" - -e "s|[@]PYTHON@|$PYTHON|g" - -e "s|[@]PIP3@|$PIP3|g" - -e "s|[@]PYPI_PKGS@|$PYPI_PKGS|g" - -e "s|[@]XML_CATALOG_FILES@|$XML_CATALOG_FILES|g" - ci/cirrus/$NAME.yml + - set +o allexport + - cirrus-vars ci/cirrus/$NAME.yml - cat ci/cirrus/$NAME.yml - cirrus-run -v --show-build-log always ci/cirrus/$NAME.yml rules: