[libvirt PATCH v3 0/4] ci: Use GitLab container registry
Changes from [v2]: * use $CI_COMMIT_REF_SLUG instead of 'master' as tag name, so that it's possible to test changes to the Dockerfiles that affect the subsequent build jobs in a feature branch; * add CentOS Stream; * rename 'preliminary_checks' stage to 'sanity_checks'. Changes from [v1]: * only build containers necessary for extra jobs when said jobs are actually going to run; * rename container build jobs to '$arch-$os-container'; * rename 'other' stage to 'preliminary_checks' and move it before native builds; * simplify build system integration. [v1] https://www.redhat.com/archives/libvir-list/2020-May/msg01183.html [v2] https://www.redhat.com/archives/libvir-list/2020-June/msg00067.html Andrea Bolognani (4): ci: Use variables to build image names ci: Add 'sanity_checks' stage ci: Use GitLab container registry ci: Update build system integration .gitlab-ci.yml | 333 ++++++++++++++++-- ci/Makefile | 10 +- ci/containers/README.rst | 14 + ci/containers/ci-centos-7.Dockerfile | 137 +++++++ ci/containers/ci-centos-8.Dockerfile | 108 ++++++ ci/containers/ci-centos-stream.Dockerfile | 109 ++++++ .../ci-debian-10-cross-aarch64.Dockerfile | 122 +++++++ .../ci-debian-10-cross-armv6l.Dockerfile | 120 +++++++ .../ci-debian-10-cross-armv7l.Dockerfile | 121 +++++++ .../ci-debian-10-cross-i686.Dockerfile | 121 +++++++ .../ci-debian-10-cross-mips.Dockerfile | 121 +++++++ .../ci-debian-10-cross-mips64el.Dockerfile | 121 +++++++ .../ci-debian-10-cross-mipsel.Dockerfile | 121 +++++++ .../ci-debian-10-cross-ppc64le.Dockerfile | 121 +++++++ .../ci-debian-10-cross-s390x.Dockerfile | 121 +++++++ ci/containers/ci-debian-10.Dockerfile | 112 ++++++ .../ci-debian-9-cross-aarch64.Dockerfile | 126 +++++++ .../ci-debian-9-cross-armv6l.Dockerfile | 124 +++++++ .../ci-debian-9-cross-armv7l.Dockerfile | 125 +++++++ .../ci-debian-9-cross-mips.Dockerfile | 125 +++++++ .../ci-debian-9-cross-mips64el.Dockerfile | 125 +++++++ .../ci-debian-9-cross-mipsel.Dockerfile | 125 +++++++ .../ci-debian-9-cross-ppc64le.Dockerfile | 125 +++++++ .../ci-debian-9-cross-s390x.Dockerfile | 125 +++++++ ci/containers/ci-debian-9.Dockerfile | 116 ++++++ .../ci-debian-sid-cross-aarch64.Dockerfile | 122 +++++++ .../ci-debian-sid-cross-armv6l.Dockerfile | 120 +++++++ .../ci-debian-sid-cross-armv7l.Dockerfile | 121 +++++++ .../ci-debian-sid-cross-i686.Dockerfile | 121 +++++++ .../ci-debian-sid-cross-mips.Dockerfile | 121 +++++++ .../ci-debian-sid-cross-mips64el.Dockerfile | 121 +++++++ .../ci-debian-sid-cross-mipsel.Dockerfile | 120 +++++++ .../ci-debian-sid-cross-ppc64le.Dockerfile | 121 +++++++ .../ci-debian-sid-cross-s390x.Dockerfile | 121 +++++++ ci/containers/ci-debian-sid.Dockerfile | 112 ++++++ ci/containers/ci-fedora-31.Dockerfile | 109 ++++++ ci/containers/ci-fedora-32.Dockerfile | 109 ++++++ ...ci-fedora-rawhide-cross-mingw32.Dockerfile | 130 +++++++ ...ci-fedora-rawhide-cross-mingw64.Dockerfile | 130 +++++++ ci/containers/ci-fedora-rawhide.Dockerfile | 110 ++++++ ci/containers/ci-opensuse-151.Dockerfile | 109 ++++++ ci/containers/ci-ubuntu-1804.Dockerfile | 117 ++++++ ci/containers/ci-ubuntu-2004.Dockerfile | 113 ++++++ ci/containers/refresh | 43 +++ ci/list-images.sh | 24 +- 45 files changed, 5173 insertions(+), 49 deletions(-) create mode 100644 ci/containers/README.rst create mode 100644 ci/containers/ci-centos-7.Dockerfile create mode 100644 ci/containers/ci-centos-8.Dockerfile create mode 100644 ci/containers/ci-centos-stream.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-aarch64.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-armv6l.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-armv7l.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-i686.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-mips.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-mips64el.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-mipsel.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-ppc64le.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-s390x.Dockerfile create mode 100644 ci/containers/ci-debian-10.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-aarch64.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-armv6l.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-armv7l.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-mips.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-mips64el.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-mipsel.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-ppc64le.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-s390x.Dockerfile create mode 100644 ci/containers/ci-debian-9.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-aarch64.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-armv6l.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-armv7l.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-i686.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-mips.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-mips64el.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-mipsel.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-ppc64le.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-s390x.Dockerfile create mode 100644 ci/containers/ci-debian-sid.Dockerfile create mode 100644 ci/containers/ci-fedora-31.Dockerfile create mode 100644 ci/containers/ci-fedora-32.Dockerfile create mode 100644 ci/containers/ci-fedora-rawhide-cross-mingw32.Dockerfile create mode 100644 ci/containers/ci-fedora-rawhide-cross-mingw64.Dockerfile create mode 100644 ci/containers/ci-fedora-rawhide.Dockerfile create mode 100644 ci/containers/ci-opensuse-151.Dockerfile create mode 100644 ci/containers/ci-ubuntu-1804.Dockerfile create mode 100644 ci/containers/ci-ubuntu-2004.Dockerfile create mode 100755 ci/containers/refresh -- 2.25.4
This removes a lot of repetition and makes the configuration much easier to read. Signed-off-by: Andrea Bolognani <abologna@redhat.com> Reviewed-by: Daniel P. Berrangé <berrange@redhat.com> --- .gitlab-ci.yml | 79 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a8142b506..8405b3b67c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ stages: # Default native build jobs that are always run .native_build_default_job_template: &native_build_default_job_definition stage: native_build + image: quay.io/libvirt/buildenv-libvirt-$NAME:latest cache: paths: - ccache/ @@ -65,6 +66,7 @@ stages: # Default cross build jobs that are always run .cross_build_default_job_template: &cross_build_default_job_definition stage: cross_build + image: quay.io/libvirt/buildenv-libvirt-$NAME-cross-$CROSS:latest cache: paths: - ccache/ @@ -90,47 +92,58 @@ stages: x64-debian-9: <<: *native_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-9:latest + variables: + NAME: debian-9 x64-debian-10: <<: *native_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-10:latest + variables: + NAME: debian-10 x64-debian-sid: <<: *native_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-sid:latest + variables: + NAME: debian-sid x64-centos-7: <<: *native_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-centos-7:latest + variables: + NAME: centos-7 x64-centos-8: <<: *native_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest + variables: + NAME: centos-8 x64-fedora-31: <<: *native_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-fedora-31:latest + variables: + NAME: fedora-31 x64-fedora-32: <<: *native_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-fedora-32:latest + variables: + NAME: fedora-32 x64-fedora-rawhide: <<: *native_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-fedora-rawhide:latest + variables: + NAME: fedora-rawhide x64-opensuse-151: <<: *native_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-opensuse-151:latest + variables: + NAME: opensuse-151 x64-ubuntu-1804: <<: *native_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-ubuntu-1804:latest + variables: + NAME: ubuntu-1804 x64-ubuntu-2004: <<: *native_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-ubuntu-2004:latest + variables: + NAME: ubuntu-2004 x64-freebsd-12-build: <<: *cirrus_build_default_job_definition @@ -147,47 +160,69 @@ x64-macos-1015-build: armv6l-debian-9: <<: *cross_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-9-cross-armv6l:latest + variables: + NAME: debian-9 + CROSS: armv6l mips64el-debian-9: <<: *cross_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-9-cross-mips64el:latest + variables: + NAME: debian-9 + CROSS: mips64el mips-debian-9: <<: *cross_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-9-cross-mips:latest + variables: + NAME: debian-9 + CROSS: mips aarch64-debian-10: <<: *cross_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-10-cross-aarch64:latest + variables: + NAME: debian-10 + CROSS: aarch64 ppc64le-debian-10: <<: *cross_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-10-cross-ppc64le:latest + variables: + NAME: debian-10 + CROSS: ppc64le s390x-debian-10: <<: *cross_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-10-cross-s390x:latest + variables: + NAME: debian-10 + CROSS: s390x armv7l-debian-sid: <<: *cross_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-sid-cross-armv7l:latest + variables: + NAME: debian-sid + CROSS: armv7l i686-debian-sid: <<: *cross_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-sid-cross-i686:latest + variables: + NAME: debian-sid + CROSS: i686 mipsel-debian-sid: <<: *cross_build_extra_job_definition - image: quay.io/libvirt/buildenv-libvirt-debian-sid-cross-mipsel:latest + variables: + NAME: debian-sid + CROSS: mipsel mingw32-fedora-rawhide: <<: *cross_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-fedora-rawhide-cross-mingw32:latest + variables: + NAME: fedora-rawhide + CROSS: mingw32 mingw64-fedora-rawhide: <<: *cross_build_default_job_definition - image: quay.io/libvirt/buildenv-libvirt-fedora-rawhide-cross-mingw64:latest + variables: + NAME: fedora-rawhide + CROSS: mingw64 # This artifact published by this job is downloaded by libvirt.org to -- 2.25.4
We're going to build container images as part of the CI pipeline soon, which means that we need to make sure all jobs that run in a container image which is not provided by an external project happens later in the pipeline; at the same time, we want to perform some basic sanity checks, such as DCO checking, as early as possible, and this new stage achieves just that. Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8405b3b67c..ba31afd205 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ variables: GIT_DEPTH: 100 stages: + - sanity_checks - prebuild - native_build - cross_build @@ -294,7 +295,7 @@ potfile: # this test on developer's personal forks from which # merge requests are submitted check-dco: - stage: prebuild + stage: sanity_checks image: registry.gitlab.com/libvirt/libvirt-ci/check-dco:master script: - /check-dco -- 2.25.4
Instead of using pre-built containers hosted on Quay, build containers as part of the GitLab CI pipeline and upload them to the GitLab container registry for later use. This will not significantly slow down builds, because containers are only rebuilt when the corresponding Dockerfile has been modified. Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- .gitlab-ci.yml | 255 +++++++++++++++++- ci/containers/README.rst | 14 + ci/containers/ci-centos-7.Dockerfile | 137 ++++++++++ ci/containers/ci-centos-8.Dockerfile | 108 ++++++++ ci/containers/ci-centos-stream.Dockerfile | 109 ++++++++ .../ci-debian-10-cross-aarch64.Dockerfile | 122 +++++++++ .../ci-debian-10-cross-armv6l.Dockerfile | 120 +++++++++ .../ci-debian-10-cross-armv7l.Dockerfile | 121 +++++++++ .../ci-debian-10-cross-i686.Dockerfile | 121 +++++++++ .../ci-debian-10-cross-mips.Dockerfile | 121 +++++++++ .../ci-debian-10-cross-mips64el.Dockerfile | 121 +++++++++ .../ci-debian-10-cross-mipsel.Dockerfile | 121 +++++++++ .../ci-debian-10-cross-ppc64le.Dockerfile | 121 +++++++++ .../ci-debian-10-cross-s390x.Dockerfile | 121 +++++++++ ci/containers/ci-debian-10.Dockerfile | 112 ++++++++ .../ci-debian-9-cross-aarch64.Dockerfile | 126 +++++++++ .../ci-debian-9-cross-armv6l.Dockerfile | 124 +++++++++ .../ci-debian-9-cross-armv7l.Dockerfile | 125 +++++++++ .../ci-debian-9-cross-mips.Dockerfile | 125 +++++++++ .../ci-debian-9-cross-mips64el.Dockerfile | 125 +++++++++ .../ci-debian-9-cross-mipsel.Dockerfile | 125 +++++++++ .../ci-debian-9-cross-ppc64le.Dockerfile | 125 +++++++++ .../ci-debian-9-cross-s390x.Dockerfile | 125 +++++++++ ci/containers/ci-debian-9.Dockerfile | 116 ++++++++ .../ci-debian-sid-cross-aarch64.Dockerfile | 122 +++++++++ .../ci-debian-sid-cross-armv6l.Dockerfile | 120 +++++++++ .../ci-debian-sid-cross-armv7l.Dockerfile | 121 +++++++++ .../ci-debian-sid-cross-i686.Dockerfile | 121 +++++++++ .../ci-debian-sid-cross-mips.Dockerfile | 121 +++++++++ .../ci-debian-sid-cross-mips64el.Dockerfile | 121 +++++++++ .../ci-debian-sid-cross-mipsel.Dockerfile | 120 +++++++++ .../ci-debian-sid-cross-ppc64le.Dockerfile | 121 +++++++++ .../ci-debian-sid-cross-s390x.Dockerfile | 121 +++++++++ ci/containers/ci-debian-sid.Dockerfile | 112 ++++++++ ci/containers/ci-fedora-31.Dockerfile | 109 ++++++++ ci/containers/ci-fedora-32.Dockerfile | 109 ++++++++ ...ci-fedora-rawhide-cross-mingw32.Dockerfile | 130 +++++++++ ...ci-fedora-rawhide-cross-mingw64.Dockerfile | 130 +++++++++ ci/containers/ci-fedora-rawhide.Dockerfile | 110 ++++++++ ci/containers/ci-opensuse-151.Dockerfile | 109 ++++++++ ci/containers/ci-ubuntu-1804.Dockerfile | 117 ++++++++ ci/containers/ci-ubuntu-2004.Dockerfile | 113 ++++++++ ci/containers/refresh | 43 +++ 43 files changed, 5105 insertions(+), 5 deletions(-) create mode 100644 ci/containers/README.rst create mode 100644 ci/containers/ci-centos-7.Dockerfile create mode 100644 ci/containers/ci-centos-8.Dockerfile create mode 100644 ci/containers/ci-centos-stream.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-aarch64.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-armv6l.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-armv7l.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-i686.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-mips.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-mips64el.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-mipsel.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-ppc64le.Dockerfile create mode 100644 ci/containers/ci-debian-10-cross-s390x.Dockerfile create mode 100644 ci/containers/ci-debian-10.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-aarch64.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-armv6l.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-armv7l.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-mips.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-mips64el.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-mipsel.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-ppc64le.Dockerfile create mode 100644 ci/containers/ci-debian-9-cross-s390x.Dockerfile create mode 100644 ci/containers/ci-debian-9.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-aarch64.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-armv6l.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-armv7l.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-i686.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-mips.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-mips64el.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-mipsel.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-ppc64le.Dockerfile create mode 100644 ci/containers/ci-debian-sid-cross-s390x.Dockerfile create mode 100644 ci/containers/ci-debian-sid.Dockerfile create mode 100644 ci/containers/ci-fedora-31.Dockerfile create mode 100644 ci/containers/ci-fedora-32.Dockerfile create mode 100644 ci/containers/ci-fedora-rawhide-cross-mingw32.Dockerfile create mode 100644 ci/containers/ci-fedora-rawhide-cross-mingw64.Dockerfile create mode 100644 ci/containers/ci-fedora-rawhide.Dockerfile create mode 100644 ci/containers/ci-opensuse-151.Dockerfile create mode 100644 ci/containers/ci-ubuntu-1804.Dockerfile create mode 100644 ci/containers/ci-ubuntu-2004.Dockerfile create mode 100755 ci/containers/refresh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba31afd205..f79f930e66 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ variables: stages: - sanity_checks + - containers - prebuild - native_build - cross_build @@ -16,10 +17,43 @@ stages: # Common templates +# Containers that are necessary for the default pipeline +.container_default_job_template: &container_default_job_definition + image: docker:stable + stage: containers + services: + - docker:dind + before_script: + - export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:$CI_COMMIT_REF_SLUG" + - export COMMON_TAG="$CI_REGISTRY/libvirt/libvirt/ci-$NAME:master" + - docker info + - docker login registry.gitlab.com -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + script: + - docker pull "$TAG" || docker pull "$COMMON_TAG" || true + - docker build --cache-from "$TAG" --cache-from "$COMMON_TAG" --tag "$TAG" -f "ci/containers/ci-$NAME.Dockerfile" ci/containers + - docker push "$TAG" + after_script: + - docker logout + +# Containers that are only needed for the full pipeline +.container_extra_job_template: &container_extra_job_definition + <<: *container_default_job_definition + only: + - master + - /^ci-full-.*$/ + +# We build many containers which can be useful to debug problems but are not +# needed for the pipeline itself to complete: those sometimes fail, and when +# that happens it's mostly because of temporary issues with Debian sid. We +# don't want those failures to affect the overall pipeline status +.container_optional_job_template: &container_optional_job_definition + <<: *container_extra_job_definition + allow_failure: true + # Default native build jobs that are always run .native_build_default_job_template: &native_build_default_job_definition stage: native_build - image: quay.io/libvirt/buildenv-libvirt-$NAME:latest + image: $CI_REGISTRY_IMAGE/ci-$NAME:$CI_COMMIT_REF_SLUG cache: paths: - ccache/ @@ -67,7 +101,7 @@ stages: # Default cross build jobs that are always run .cross_build_default_job_template: &cross_build_default_job_definition stage: cross_build - image: quay.io/libvirt/buildenv-libvirt-$NAME-cross-$CROSS:latest + image: $CI_REGISTRY_IMAGE/ci-$NAME-cross-$CROSS:$CI_COMMIT_REF_SLUG cache: paths: - ccache/ @@ -89,6 +123,212 @@ stages: - /^ci-full-.*$/ +# Native container build jobs + +x64-centos-7-container: + <<: *container_default_job_definition + variables: + NAME: centos-7 + +x64-centos-8-container: + <<: *container_default_job_definition + variables: + NAME: centos-8 + +x64-centos-stream-container: + <<: *container_extra_job_definition + variables: + NAME: centos-stream + +x64-debian-9-container: + <<: *container_extra_job_definition + variables: + NAME: debian-9 + +x64-debian-10-container: + <<: *container_default_job_definition + variables: + NAME: debian-10 + +x64-debian-sid-container: + <<: *container_extra_job_definition + variables: + NAME: debian-sid + +x64-fedora-31-container: + <<: *container_extra_job_definition + variables: + NAME: fedora-31 + +x64-fedora-32-container: + <<: *container_default_job_definition + variables: + NAME: fedora-32 + +x64-fedora-rawhide-container: + <<: *container_default_job_definition + variables: + NAME: fedora-rawhide + +x64-opensuse-151-container: + <<: *container_default_job_definition + variables: + NAME: opensuse-151 + +x64-ubuntu-1804-container: + <<: *container_extra_job_definition + variables: + NAME: ubuntu-1804 + +x64-ubuntu-2004-container: + <<: *container_default_job_definition + variables: + NAME: ubuntu-2004 + + +# Cross-build containers build jobs + +aarch64-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-aarch64 + +armv6l-debian-9-container: + <<: *container_extra_job_definition + variables: + NAME: debian-9-cross-armv6l + +armv7l-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-armv7l + +mips-debian-9-container: + <<: *container_extra_job_definition + variables: + NAME: debian-9-cross-mips + +mips64el-debian-9-container: + <<: *container_extra_job_definition + variables: + NAME: debian-9-cross-mips64el + +mipsel-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-mipsel + +ppc64le-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-ppc64le + +s390x-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-s390x + +aarch64-debian-10-container: + <<: *container_extra_job_definition + variables: + NAME: debian-10-cross-aarch64 + +armv6l-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-armv6l + +armv7-ldebian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-armv7l + +i686-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-i686 + +mips-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-mips + +mips64el-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-mips64el + +mipsel-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-mipsel + +ppc64le-debian-10-container: + <<: *container_extra_job_definition + variables: + NAME: debian-10-cross-ppc64le + +s390x-debian-10-container: + <<: *container_default_job_definition + variables: + NAME: debian-10-cross-s390x + +aarch64-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-aarch64 + +armv6l-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-armv6l + +armv7-ldebian-sid-container: + <<: *container_default_job_definition + variables: + NAME: debian-sid-cross-armv7l + +i686-debian-sid-container: + <<: *container_extra_job_definition + variables: + NAME: debian-sid-cross-i686 + +mips-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-mips + +mips64el-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-mips64el + +mipsel-debian-sid-container: + <<: *container_extra_job_definition + variables: + NAME: debian-sid-cross-mipsel + +ppc64le-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-ppc64le + +s390x-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-s390x + +mingw32-fedora-rawhide-container: + <<: *container_default_job_definition + variables: + NAME: fedora-rawhide-cross-mingw32 + +mingw64-fedora-rawhide-container: + <<: *container_default_job_definition + variables: + NAME: fedora-rawhide-cross-mingw64 + + # Native architecture build + test jobs x64-debian-9: @@ -116,6 +356,11 @@ x64-centos-8: variables: NAME: centos-8 +x64-centos-stream: + <<: *native_build_extra_job_definition + variables: + NAME: centos-stream + x64-fedora-31: <<: *native_build_extra_job_definition variables: @@ -231,6 +476,7 @@ mingw64-fedora-rawhide: # https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=webs... website: stage: prebuild + image: $CI_REGISTRY_IMAGE/ci-centos-8:$CI_COMMIT_REF_SLUG before_script: - *script_variables script: @@ -241,7 +487,6 @@ website: - $MAKE -C docs install - cd .. - mv vroot/share/doc/libvirt/html/ website - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest artifacts: expose_as: 'Website' name: 'website' @@ -253,6 +498,7 @@ website: codestyle: stage: prebuild + image: $CI_REGISTRY_IMAGE/ci-centos-8:$CI_COMMIT_REF_SLUG before_script: - *script_variables script: @@ -260,7 +506,6 @@ codestyle: - cd build - ../autogen.sh || (cat config.log && exit 1) - $MAKE syntax-check - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest # This artifact published by this job is downloaded to push to Weblate @@ -268,6 +513,7 @@ codestyle: # https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=potf... potfile: stage: prebuild + image: $CI_REGISTRY_IMAGE/ci-centos-8:$CI_COMMIT_REF_SLUG only: - master before_script: @@ -280,7 +526,6 @@ potfile: - $MAKE -C po libvirt.pot - cd .. - cp po/libvirt.pot libvirt.pot - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest artifacts: expose_as: 'Potfile' name: 'potfile' diff --git a/ci/containers/README.rst b/ci/containers/README.rst new file mode 100644 index 0000000000..530897e311 --- /dev/null +++ b/ci/containers/README.rst @@ -0,0 +1,14 @@ +CI job assets +============= + +This directory contains assets used in the automated CI jobs, most +notably the Dockerfiles used to build container images in which the +CI jobs then run. + +The ``refresh`` script is used to re-create the Dockerfiles using the +``lcitool`` command that is provided by repo +https://gitlab.com/libvirt/libvirt-ci + +The containers are built during the CI process and cached in the GitLab +container registry of the project doing the build. The cached containers +can be deleted at any time and will be correctly rebuilt. [... imagine lots and lots of Dockerfiles here ...] diff --git a/ci/containers/refresh b/ci/containers/refresh new file mode 100755 index 0000000000..8c00363ae1 --- /dev/null +++ b/ci/containers/refresh @@ -0,0 +1,43 @@ +#!/bin/sh + +if test -z "$1" +then + echo "syntax: $0 PATH-TO-LCITOOL" + exit 1 +fi + +LCITOOL=$1 + +if ! test -x "$LCITOOL" +then + echo "$LCITOOL is not executable" + exit 1 +fi + +HOSTS=$($LCITOOL hosts | grep -v freebsd) + +for host in $HOSTS +do + name=${host#libvirt-} + + case "$name" in + fedora-rawhide) + for cross in mingw32 mingw64 + do + $LCITOOL dockerfile $host libvirt --cross $cross >ci-$name-cross-$cross.Dockerfile + done + ;; + debian-*) + for cross in aarch64 armv6l armv7l i686 mips mips64el mipsel ppc64le s390x + do + if test "$name" = "debian-9" && test "$cross" = "i686" + then + continue + fi + $LCITOOL dockerfile $host libvirt --cross $cross >ci-$name-cross-$cross.Dockerfile + done + ;; + esac + + $LCITOOL dockerfile $host libvirt >ci-$name.Dockerfile +done -- 2.25.4
On Wed, Jun 10, 2020 at 05:34:13PM +0200, Andrea Bolognani wrote:
Instead of using pre-built containers hosted on Quay, build containers as part of the GitLab CI pipeline and upload them to the GitLab container registry for later use.
This will not significantly slow down builds, because containers are only rebuilt when the corresponding Dockerfile has been modified.
Signed-off-by: Andrea Bolognani <abologna@redhat.com> diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba31afd205..f79f930e66 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ variables:
stages: - sanity_checks + - containers - prebuild - native_build - cross_build @@ -16,10 +17,43 @@ stages:
# Common templates
+# Containers that are necessary for the default pipeline +.container_default_job_template: &container_default_job_definition + image: docker:stable + stage: containers + services: + - docker:dind + before_script: + - export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:$CI_COMMIT_REF_SLUG" + - export COMMON_TAG="$CI_REGISTRY/libvirt/libvirt/ci-$NAME:master"
This is different to what we've done on all the other repos. I originally used this, but noted that it results in a ever growing set of tags being published in the container registry, as users will have a new branch name for every piece of work. It also means you'll never a get a cache hit from the user's registry across feature branches, though that is mitigated to by fact that we'll consider the global cache too I guess.
+ - docker info + - docker login registry.gitlab.com -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + script: + - docker pull "$TAG" || docker pull "$COMMON_TAG" || true + - docker build --cache-from "$TAG" --cache-from "$COMMON_TAG" --tag "$TAG" -f "ci/containers/ci-$NAME.Dockerfile" ci/containers + - docker push "$TAG" + after_script: + - docker logout
+ +# Containers that are only needed for the full pipeline +.container_extra_job_template: &container_extra_job_definition + <<: *container_default_job_definition + only: + - master + - /^ci-full-.*$/ + +# We build many containers which can be useful to debug problems but are not +# needed for the pipeline itself to complete: those sometimes fail, and when +# that happens it's mostly because of temporary issues with Debian sid. We +# don't want those failures to affect the overall pipeline status +.container_optional_job_template: &container_optional_job_definition + <<: *container_extra_job_definition + allow_failure: true + # Default native build jobs that are always run .native_build_default_job_template: &native_build_default_job_definition stage: native_build - image: quay.io/libvirt/buildenv-libvirt-$NAME:latest + image: $CI_REGISTRY_IMAGE/ci-$NAME:$CI_COMMIT_REF_SLUG cache: paths: - ccache/ @@ -67,7 +101,7 @@ stages: # Default cross build jobs that are always run .cross_build_default_job_template: &cross_build_default_job_definition stage: cross_build - image: quay.io/libvirt/buildenv-libvirt-$NAME-cross-$CROSS:latest + image: $CI_REGISTRY_IMAGE/ci-$NAME-cross-$CROSS:$CI_COMMIT_REF_SLUG cache: paths: - ccache/ @@ -89,6 +123,212 @@ stages: - /^ci-full-.*$/
+# Native container build jobs + +x64-centos-7-container: + <<: *container_default_job_definition + variables: + NAME: centos-7 + +x64-centos-8-container: + <<: *container_default_job_definition + variables: + NAME: centos-8 + +x64-centos-stream-container: + <<: *container_extra_job_definition + variables: + NAME: centos-stream + +x64-debian-9-container: + <<: *container_extra_job_definition + variables: + NAME: debian-9 + +x64-debian-10-container: + <<: *container_default_job_definition + variables: + NAME: debian-10 + +x64-debian-sid-container: + <<: *container_extra_job_definition + variables: + NAME: debian-sid + +x64-fedora-31-container: + <<: *container_extra_job_definition + variables: + NAME: fedora-31 + +x64-fedora-32-container: + <<: *container_default_job_definition + variables: + NAME: fedora-32 + +x64-fedora-rawhide-container: + <<: *container_default_job_definition + variables: + NAME: fedora-rawhide + +x64-opensuse-151-container: + <<: *container_default_job_definition + variables: + NAME: opensuse-151 + +x64-ubuntu-1804-container: + <<: *container_extra_job_definition + variables: + NAME: ubuntu-1804 + +x64-ubuntu-2004-container: + <<: *container_default_job_definition + variables: + NAME: ubuntu-2004 + + +# Cross-build containers build jobs + +aarch64-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-aarch64 + +armv6l-debian-9-container: + <<: *container_extra_job_definition + variables: + NAME: debian-9-cross-armv6l + +armv7l-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-armv7l + +mips-debian-9-container: + <<: *container_extra_job_definition + variables: + NAME: debian-9-cross-mips + +mips64el-debian-9-container: + <<: *container_extra_job_definition + variables: + NAME: debian-9-cross-mips64el + +mipsel-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-mipsel + +ppc64le-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-ppc64le + +s390x-debian-9-container: + <<: *container_optional_job_definition + variables: + NAME: debian-9-cross-s390x + +aarch64-debian-10-container: + <<: *container_extra_job_definition + variables: + NAME: debian-10-cross-aarch64 + +armv6l-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-armv6l + +armv7-ldebian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-armv7l + +i686-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-i686 + +mips-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-mips + +mips64el-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-mips64el + +mipsel-debian-10-container: + <<: *container_optional_job_definition + variables: + NAME: debian-10-cross-mipsel + +ppc64le-debian-10-container: + <<: *container_extra_job_definition + variables: + NAME: debian-10-cross-ppc64le + +s390x-debian-10-container: + <<: *container_default_job_definition + variables: + NAME: debian-10-cross-s390x + +aarch64-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-aarch64 + +armv6l-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-armv6l + +armv7-ldebian-sid-container: + <<: *container_default_job_definition + variables: + NAME: debian-sid-cross-armv7l + +i686-debian-sid-container: + <<: *container_extra_job_definition + variables: + NAME: debian-sid-cross-i686 + +mips-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-mips + +mips64el-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-mips64el + +mipsel-debian-sid-container: + <<: *container_extra_job_definition + variables: + NAME: debian-sid-cross-mipsel + +ppc64le-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-ppc64le + +s390x-debian-sid-container: + <<: *container_optional_job_definition + variables: + NAME: debian-sid-cross-s390x + +mingw32-fedora-rawhide-container: + <<: *container_default_job_definition + variables: + NAME: fedora-rawhide-cross-mingw32 + +mingw64-fedora-rawhide-container: + <<: *container_default_job_definition + variables: + NAME: fedora-rawhide-cross-mingw64 + + # Native architecture build + test jobs
x64-debian-9: @@ -116,6 +356,11 @@ x64-centos-8: variables: NAME: centos-8
+x64-centos-stream: + <<: *native_build_extra_job_definition + variables: + NAME: centos-stream + x64-fedora-31: <<: *native_build_extra_job_definition variables: @@ -231,6 +476,7 @@ mingw64-fedora-rawhide: # https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=webs... website: stage: prebuild + image: $CI_REGISTRY_IMAGE/ci-centos-8:$CI_COMMIT_REF_SLUG before_script: - *script_variables script: @@ -241,7 +487,6 @@ website: - $MAKE -C docs install - cd .. - mv vroot/share/doc/libvirt/html/ website - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest artifacts: expose_as: 'Website' name: 'website' @@ -253,6 +498,7 @@ website:
codestyle: stage: prebuild + image: $CI_REGISTRY_IMAGE/ci-centos-8:$CI_COMMIT_REF_SLUG before_script: - *script_variables script: @@ -260,7 +506,6 @@ codestyle: - cd build - ../autogen.sh || (cat config.log && exit 1) - $MAKE syntax-check - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest
# This artifact published by this job is downloaded to push to Weblate @@ -268,6 +513,7 @@ codestyle: # https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=potf... potfile: stage: prebuild + image: $CI_REGISTRY_IMAGE/ci-centos-8:$CI_COMMIT_REF_SLUG only: - master before_script: @@ -280,7 +526,6 @@ potfile: - $MAKE -C po libvirt.pot - cd .. - cp po/libvirt.pot libvirt.pot - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest artifacts: expose_as: 'Potfile' name: 'potfile' diff --git a/ci/containers/README.rst b/ci/containers/README.rst new file mode 100644 index 0000000000..530897e311 --- /dev/null +++ b/ci/containers/README.rst @@ -0,0 +1,14 @@ +CI job assets +============= + +This directory contains assets used in the automated CI jobs, most +notably the Dockerfiles used to build container images in which the +CI jobs then run. + +The ``refresh`` script is used to re-create the Dockerfiles using the +``lcitool`` command that is provided by repo +https://gitlab.com/libvirt/libvirt-ci + +The containers are built during the CI process and cached in the GitLab +container registry of the project doing the build. The cached containers +can be deleted at any time and will be correctly rebuilt.
[... imagine lots and lots of Dockerfiles here ...]
diff --git a/ci/containers/refresh b/ci/containers/refresh new file mode 100755 index 0000000000..8c00363ae1 --- /dev/null +++ b/ci/containers/refresh @@ -0,0 +1,43 @@ +#!/bin/sh + +if test -z "$1" +then + echo "syntax: $0 PATH-TO-LCITOOL" + exit 1 +fi + +LCITOOL=$1 + +if ! test -x "$LCITOOL" +then + echo "$LCITOOL is not executable" + exit 1 +fi + +HOSTS=$($LCITOOL hosts | grep -v freebsd) + +for host in $HOSTS +do + name=${host#libvirt-} + + case "$name" in + fedora-rawhide) + for cross in mingw32 mingw64 + do + $LCITOOL dockerfile $host libvirt --cross $cross >ci-$name-cross-$cross.Dockerfile + done + ;; + debian-*) + for cross in aarch64 armv6l armv7l i686 mips mips64el mipsel ppc64le s390x + do + if test "$name" = "debian-9" && test "$cross" = "i686" + then + continue + fi + $LCITOOL dockerfile $host libvirt --cross $cross >ci-$name-cross-$cross.Dockerfile + done + ;; + esac + + $LCITOOL dockerfile $host libvirt >ci-$name.Dockerfile
In all the other projects we've just used $host.Dockerfile. If we really want to eliminate the "libvirt-" prefix, then it is probably better to modify lcitool to remove this prefix. It is a pretty pointless name prefix considering the targets are used for non-libvirt projects too. Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On Wed, 2020-06-10 at 17:11 +0100, Daniel P. Berrangé wrote:
On Wed, Jun 10, 2020 at 05:34:13PM +0200, Andrea Bolognani wrote:
+.container_default_job_template: &container_default_job_definition + image: docker:stable + stage: containers + services: + - docker:dind + before_script: + - export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:$CI_COMMIT_REF_SLUG" + - export COMMON_TAG="$CI_REGISTRY/libvirt/libvirt/ci-$NAME:master"
This is different to what we've done on all the other repos. I originally used this, but noted that it results in a ever growing set of tags being published in the container registry, as users will have a new branch name for every piece of work. It also means you'll never a get a cache hit from the user's registry across feature branches, though that is mitigated to by fact that we'll consider the global cache too I guess.
We can have an additional --cache-from $CI_REGISTRY_IMAGE/ci-$NAME:master to further reduce the possibility of getting a cache miss. Note that you can configure an expiration policy for tags https://docs.gitlab.com/ee/user/packages/container_registry/#managing-projec... but apparently it has to happen on a per-project basis instead of being something that you can set globally for your account. Is having a lot of tags such a big problem? It seems like it's not unusual... See https://hub.docker.com/_/nginx?tab=tags for an extreme example. But yeah, maybe I'm overthinking this. If the pipeline produces the containers it consumes, then whether you label them as "latest" or "master" or "a0sd90lv_k1" doesn't really make any difference, because the next pipeline is going to build the containers again before using them. There is still one scenario in which reusing the same name could lead to unwanted results, however, and that is when two or more pipelines are running at the same time. Right now this is allowed, but by using resource groups https://docs.gitlab.com/ee/ci/yaml/#resource_group we should be able to prevent that from happening.
+ $LCITOOL dockerfile $host libvirt >ci-$name.Dockerfile
In all the other projects we've just used $host.Dockerfile.
If we really want to eliminate the "libvirt-" prefix, then it is probably better to modify lcitool to remove this prefix. It is a pretty pointless name prefix considering the targets are used for non-libvirt projects too.
I agree that it looks silly at this point, and I've idly considered getting rid of it, but as long as we keep the CentOS CI environment running it needs to stay because workers there are not namespaced, so the prefix can't be removed. In the meantime, I can skip the pre-processing and keep this more consistent with what goes on in the other repositories. -- Andrea Bolognani / Red Hat / Virtualization
On Wed, Jun 10, 2020 at 07:29:53PM +0200, Andrea Bolognani wrote:
On Wed, 2020-06-10 at 17:11 +0100, Daniel P. Berrangé wrote:
On Wed, Jun 10, 2020 at 05:34:13PM +0200, Andrea Bolognani wrote:
+.container_default_job_template: &container_default_job_definition + image: docker:stable + stage: containers + services: + - docker:dind + before_script: + - export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:$CI_COMMIT_REF_SLUG" + - export COMMON_TAG="$CI_REGISTRY/libvirt/libvirt/ci-$NAME:master"
This is different to what we've done on all the other repos. I originally used this, but noted that it results in a ever growing set of tags being published in the container registry, as users will have a new branch name for every piece of work. It also means you'll never a get a cache hit from the user's registry across feature branches, though that is mitigated to by fact that we'll consider the global cache too I guess.
We can have an additional
--cache-from $CI_REGISTRY_IMAGE/ci-$NAME:master
to further reduce the possibility of getting a cache miss.
Note that you can configure an expiration policy for tags
https://docs.gitlab.com/ee/user/packages/container_registry/#managing-projec...
but apparently it has to happen on a per-project basis instead of being something that you can set globally for your account.
Is having a lot of tags such a big problem? It seems like it's not unusual... See
https://hub.docker.com/_/nginx?tab=tags
for an extreme example.
But yeah, maybe I'm overthinking this. If the pipeline produces the containers it consumes, then whether you label them as "latest" or "master" or "a0sd90lv_k1" doesn't really make any difference, because the next pipeline is going to build the containers again before using them.
I just never much like the idea of things growing without bounds, even if someone else is paying for storage.
There is still one scenario in which reusing the same name could lead to unwanted results, however, and that is when two or more pipelines are running at the same time. Right now this is allowed, but by using resource groups
https://docs.gitlab.com/ee/ci/yaml/#resource_group
we should be able to prevent that from happening.
NB, running multiple pipelines in parallel is only a problem if those pipelines actually contain changes to the dockerfile recipes. Otherwise the rebuild of container images is essentially a no-op. Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
On Thu, 2020-06-11 at 10:22 +0100, Daniel P. Berrangé wrote:
On Wed, Jun 10, 2020 at 07:29:53PM +0200, Andrea Bolognani wrote:
But yeah, maybe I'm overthinking this. If the pipeline produces the containers it consumes, then whether you label them as "latest" or "master" or "a0sd90lv_k1" doesn't really make any difference, because the next pipeline is going to build the containers again before using them.
I just never much like the idea of things growing without bounds, even if someone else is paying for storage.
That's fair.
There is still one scenario in which reusing the same name could lead to unwanted results, however, and that is when two or more pipelines are running at the same time. Right now this is allowed, but by using resource groups
https://docs.gitlab.com/ee/ci/yaml/#resource_group
we should be able to prevent that from happening.
NB, running multiple pipelines in parallel is only a problem if those pipelines actually contain changes to the dockerfile recipes. Otherwise the rebuild of container images is essentially a no-op.
Yeah. I posted a v4 which doesn't really do anything to prevent this scenario from happening: if that turns our to be a real problem in the future, we can introduce resource groups then. -- Andrea Bolognani / Red Hat / Virtualization
The ci-* targets need to know where our container images are stored and how they are called to work, so now that we use the GitLab container registry instead of Quay some changes are necessary. Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- ci/Makefile | 10 +++++----- ci/list-images.sh | 24 ++++++------------------ 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/ci/Makefile b/ci/Makefile index bc1dac11e3..dc8012f33b 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -50,11 +50,11 @@ CI_BUILD_SCRIPT = $(CI_ROOTDIR)/build.sh # Location of the container images we're going to pull # Can be useful to overridde to use a locally built # image instead -CI_IMAGE_PREFIX = quay.io/libvirt/buildenv-libvirt- +CI_IMAGE_PREFIX = registry.gitlab.com/libvirt/libvirt/ci- -# The default tag is ':latest' but if the container +# The default tag is ':master' but if the container # repo above uses different conventions this can override it -CI_IMAGE_TAG = :latest +CI_IMAGE_TAG = :master # We delete the virtual root after completion, set # to 0 if you need to keep it around for debugging @@ -243,11 +243,11 @@ ci-list-images: @echo @echo "Available x86 container images:" @echo - @sh list-images.sh "$(CI_ENGINE)" "$(CI_IMAGE_PREFIX)" | grep -v cross + @sh list-images.sh "$(CI_IMAGE_PREFIX)" | grep -v cross @echo @echo "Available cross-compiler container images:" @echo - @sh list-images.sh "$(CI_ENGINE)" "$(CI_IMAGE_PREFIX)" | grep cross + @sh list-images.sh "$(CI_IMAGE_PREFIX)" | grep cross @echo ci-help: diff --git a/ci/list-images.sh b/ci/list-images.sh index 35efdb6982..b85b132253 100644 --- a/ci/list-images.sh +++ b/ci/list-images.sh @@ -1,26 +1,14 @@ #!/bin/sh -engine="$1" -prefix="$2" +prefix="${1##registry.gitlab.com/}" -do_podman() { - # Podman freaks out if the search term ends with a dash, which ours - # by default does, so let's strip it. The repository name is the - # second field in the output, and it already starts with the registry - podman search --limit 100 "${prefix%-}" | while read _ repo _; do - echo "$repo" - done -} +PROJECT_ID=192693 -do_docker() { - # Docker doesn't include the registry name in the output, so we have - # to add it. The repository name is the first field in the output - registry="${prefix%%/*}" - docker search --limit 100 "$prefix" | while read repo _; do - echo "$registry/$repo" - done +all_repos() { + curl -s "https://gitlab.com/api/v4/projects/$PROJECT_ID/registry/repositories?per_pag..." \ + | tr , '\n' | grep '"path":' | sed 's,"path":",,g;s,"$,,g' } -"do_$engine" | grep "^$prefix" | sed "s,^$prefix,,g" | while read repo; do +all_repos | grep "^$prefix" | sed "s,^$prefix,,g" | while read repo; do echo " $repo" done | sort -u -- 2.25.4
participants (2)
-
Andrea Bolognani -
Daniel P. Berrangé