This introduces CI jobs that replace the current jobs used on Jenkins for every platform except FreeBSD. A merge request workflow requires the user to fork the primary git repo into their personal namespace. In general the changes need to be tested against the current libvirt git master. If the user has a fork of the main libvirt repo, we don't want to use that by default as it may be out of date. The general goal is that the CI jobs are self-contained and don't depend on the build artifacts from the libvirt repo. We also want to avoid having an explicit dependency on the libvirt-ci repo, or on the Quay.io service. Contributors to the Perl module need to be able to make code changes which imply CI environment changes and be able to test them in isolation. Thus, the dockerfile recipes for each distro are added in the ci/ sub-directory. The first stage of the CI jobs is to use these recipes to build and publish a container image. These images are then used in the second stage to perform the actual build. The container image build is cached, inheriting from both the primary libvirt project namespace, and the user's private project namespace. Thus the performance hit of building container images will only be felt the first time the project is forked, or when the parent Docker images are rebuilt. The dockerfiles were originally generated using lcitool, but if the user makes a change that introduces new build dependencies, the corresponding packages can be added to the dockerfile recipes directly in the same commit. The change can be propagated back into the libvirt-ci.git repo asynchronously. The build job will do a minimal(-ish) build of libvirt git master and then build the rest of the code against that. Ideally the main libvirt configure script would have a way to request a minimal build of just the API and test driver, but for now we settle for just --without-libvirt which culls a large number of the drivers fairly easily. Signed-off-by: Daniel P. Berrangé <berrange(a)redhat.com&gt; --- .gitlab-ci.yml | 144 ++++++++++++++++++++++++++++++++++ ci/libvirt-centos-7.dkr | 12 +++ ci/libvirt-centos-8.dkr | 13 +++ ci/libvirt-debian-10.dkr | 14 ++++ ci/libvirt-debian-9.dkr | 14 ++++ ci/libvirt-debian-sid.dkr | 14 ++++ ci/libvirt-fedora-30.dkr | 13 +++ ci/libvirt-fedora-31.dkr | 13 +++ ci/libvirt-fedora-rawhide.dkr | 13 +++ ci/libvirt-opensuse-151.dkr | 11 +++ ci/libvirt-ubuntu-1604.dkr | 14 ++++ ci/libvirt-ubuntu-1804.dkr | 14 ++++ ci/refresh | 16 ++++ 13 files changed, 305 insertions(+) create mode 100644 ci/libvirt-centos-7.dkr create mode 100644 ci/libvirt-centos-8.dkr create mode 100644 ci/libvirt-debian-10.dkr create mode 100644 ci/libvirt-debian-9.dkr create mode 100644 ci/libvirt-debian-sid.dkr create mode 100644 ci/libvirt-fedora-30.dkr create mode 100644 ci/libvirt-fedora-31.dkr create mode 100644 ci/libvirt-fedora-rawhide.dkr create mode 100644 ci/libvirt-opensuse-151.dkr create mode 100644 ci/libvirt-ubuntu-1604.dkr create mode 100644 ci/libvirt-ubuntu-1804.dkr create mode 100755 ci/refresh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 50dae92..48c7840 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,50 @@ stages: - prebuild + - containers + - build + +.container_job_template: &container_job_definition + image: docker:stable + stage: containers + services: + - docker:dind + before_script: + - export TAG="${CI_REGISTRY_IMAGE}/buildenv-${NAME}:latest" + - export COMMON_TAG="${CI_REGISTRY}/libvirt/libvirt-perl/buildenv-${NAME}:latest" + - docker info + - docker login registry.gitlab.com -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} + script: + - docker pull ${TAG} || docker pull ${COMMON_TAG} || true + - docker build --cache-from ${TAG} --cache-from ${COMMON_TAG} --tag ${TAG} -f ci/libvirt-${NAME}.dkr ci + - docker push ${TAG} + after_script: + - docker logout + +.build_job_template: &build_job_definition + stage: build + before_script: + - export MAKEFLAGS="-j$(getconf _NPROCESSORS_ONLN)" + - export SCRATCH_DIR="/tmp/scratch" + - export VROOT="${SCRATCH_DIR}/vroot" + - export LD_LIBRARY_PATH="${VROOT}/lib" + - export PATH="${PATH}:${VROOT}/bin" + - export PKG_CONFIG_PATH="${VROOT}/lib/pkgconfig" + - export TEST_MAINTAINER=1 + - export CONFIGURE_ARGS="--without-libvirtd" + script: + - pushd ${PWD} + - mkdir -p ${SCRATCH_DIR} + - cd ${SCRATCH_DIR} + - git clone --depth 1 https://gitlab.com/libvirt/libvirt.git src + - mkdir build + - cd build + - ../src/autogen.sh --prefix=${VROOT} ${CONFIGURE_ARGS} + - make install + - popd + - perl Build.PL + - perl Build + - perl Build test # Check that all commits are signed-off for the DCO. # Skip on "libvirt" namespace, since we only need to run @@ -14,3 +58,103 @@ check-dco: except: variables: - $CI_PROJECT_NAMESPACE == 'libvirt' + +ctr-centos-7: + <<: *container_job_definition + variables: + NAME: centos-7 + +ctr-centos-8: + <<: *container_job_definition + variables: + NAME: centos-8 + +ctr-debian-9: + <<: *container_job_definition + variables: + NAME: debian-9 + +ctr-debian-10: + <<: *container_job_definition + variables: + NAME: debian-10 + +ctr-debian-sid: + <<: *container_job_definition + variables: + NAME: debian-sid + +ctr-fedora-30: + <<: *container_job_definition + variables: + NAME: fedora-30 + +ctr-fedora-31: + <<: *container_job_definition + variables: + NAME: fedora-31 + +ctr-fedora-rawhide: + <<: *container_job_definition + variables: + NAME: fedora-rawhide + +ctr-opensuse-151: + <<: *container_job_definition + variables: + NAME: opensuse-151 + +ctr-ubuntu-1604: + <<: *container_job_definition + variables: + NAME: ubuntu-1604 + +ctr-ubuntu-1804: + <<: *container_job_definition + variables: + NAME: ubuntu-1804 + + +build-centos-7: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-centos-7:latest + +build-centos-8: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-centos-8:latest + +build-debian-9: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-debian-9:latest + +build-debian-10: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-debian-10:latest + +build-debian-sid: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-debian-sid:latest + +build-fedora-30: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-fedora-30:latest + +build-fedora-31: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-fedora-31:latest + +build-fedora-rawhide: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-fedora-rawhide:latest + +build-opensuse-151: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-opensuse-151:latest + +build-ubuntu-1604: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-ubuntu-1604:latest + +build-ubuntu-1804: + <<: *build_job_definition + image: ${CI_REGISTRY_IMAGE}/buildenv-ubuntu-1804:latest diff --git a/ci/libvirt-centos-7.dkr b/ci/libvirt-centos-7.dkr new file mode 100644 index 0000000..498f20b --- /dev/null +++ b/ci/libvirt-centos-7.dkr @@ -0,0 +1,12 @@ +FROM quay.io/libvirt/buildenv-libvirt-centos-7 + +RUN yum install -y \ + perl-Archive-Tar \ + perl-CPAN-Changes \ + perl-Module-Build \ + perl-Test-Pod \ + perl-Test-Pod-Coverage \ + perl-Time-HiRes \ + perl-XML-XPath && \ + yum autoremove -y && \ + yum clean all -y diff --git a/ci/libvirt-centos-8.dkr b/ci/libvirt-centos-8.dkr new file mode 100644 index 0000000..f468e30 --- /dev/null +++ b/ci/libvirt-centos-8.dkr @@ -0,0 +1,13 @@ +FROM quay.io/libvirt/buildenv-libvirt-centos-8 + +RUN dnf install -y \ + perl-Archive-Tar \ + perl-CPAN-Changes \ + perl-Module-Build \ + perl-Test-Pod \ + perl-Test-Pod-Coverage \ + perl-Time-HiRes \ + perl-XML-XPath \ + perl-generators && \ + dnf autoremove -y && \ + dnf clean all -y diff --git a/ci/libvirt-debian-10.dkr b/ci/libvirt-debian-10.dkr new file mode 100644 index 0000000..c68a55b --- /dev/null +++ b/ci/libvirt-debian-10.dkr @@ -0,0 +1,14 @@ +FROM quay.io/libvirt/buildenv-libvirt-debian-10 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + libcpan-changes-perl \ + libmodule-build-perl \ + libtest-pod-coverage-perl \ + libtest-pod-perl \ + libtime-hr-perl \ + libxml-xpath-perl \ + perl && \ + apt-get autoremove -y && \ + apt-get autoclean -y diff --git a/ci/libvirt-debian-9.dkr b/ci/libvirt-debian-9.dkr new file mode 100644 index 0000000..74ef949 --- /dev/null +++ b/ci/libvirt-debian-9.dkr @@ -0,0 +1,14 @@ +FROM quay.io/libvirt/buildenv-libvirt-debian-9 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + libcpan-changes-perl \ + libmodule-build-perl \ + libtest-pod-coverage-perl \ + libtest-pod-perl \ + libtime-hr-perl \ + libxml-xpath-perl \ + perl && \ + apt-get autoremove -y && \ + apt-get autoclean -y diff --git a/ci/libvirt-debian-sid.dkr b/ci/libvirt-debian-sid.dkr new file mode 100644 index 0000000..8cf6ac3 --- /dev/null +++ b/ci/libvirt-debian-sid.dkr @@ -0,0 +1,14 @@ +FROM quay.io/libvirt/buildenv-libvirt-debian-sid + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + libcpan-changes-perl \ + libmodule-build-perl \ + libtest-pod-coverage-perl \ + libtest-pod-perl \ + libtime-hr-perl \ + libxml-xpath-perl \ + perl && \ + apt-get autoremove -y && \ + apt-get autoclean -y diff --git a/ci/libvirt-fedora-30.dkr b/ci/libvirt-fedora-30.dkr new file mode 100644 index 0000000..2715a6c --- /dev/null +++ b/ci/libvirt-fedora-30.dkr @@ -0,0 +1,13 @@ +FROM quay.io/libvirt/buildenv-libvirt-fedora-30 + +RUN dnf install -y \ + perl-Archive-Tar \ + perl-CPAN-Changes \ + perl-Module-Build \ + perl-Test-Pod \ + perl-Test-Pod-Coverage \ + perl-Time-HiRes \ + perl-XML-XPath \ + perl-generators && \ + dnf autoremove -y && \ + dnf clean all -y diff --git a/ci/libvirt-fedora-31.dkr b/ci/libvirt-fedora-31.dkr new file mode 100644 index 0000000..388ea3f --- /dev/null +++ b/ci/libvirt-fedora-31.dkr @@ -0,0 +1,13 @@ +FROM quay.io/libvirt/buildenv-libvirt-fedora-31 + +RUN dnf install -y \ + perl-Archive-Tar \ + perl-CPAN-Changes \ + perl-Module-Build \ + perl-Test-Pod \ + perl-Test-Pod-Coverage \ + perl-Time-HiRes \ + perl-XML-XPath \ + perl-generators && \ + dnf autoremove -y && \ + dnf clean all -y diff --git a/ci/libvirt-fedora-rawhide.dkr b/ci/libvirt-fedora-rawhide.dkr new file mode 100644 index 0000000..12b4b9e --- /dev/null +++ b/ci/libvirt-fedora-rawhide.dkr @@ -0,0 +1,13 @@ +FROM quay.io/libvirt/buildenv-libvirt-fedora-rawhide + +RUN dnf install -y \ + perl-Archive-Tar \ + perl-CPAN-Changes \ + perl-Module-Build \ + perl-Test-Pod \ + perl-Test-Pod-Coverage \ + perl-Time-HiRes \ + perl-XML-XPath \ + perl-generators && \ + dnf autoremove -y && \ + dnf clean all -y diff --git a/ci/libvirt-opensuse-151.dkr b/ci/libvirt-opensuse-151.dkr new file mode 100644 index 0000000..e423704 --- /dev/null +++ b/ci/libvirt-opensuse-151.dkr @@ -0,0 +1,11 @@ +FROM quay.io/libvirt/buildenv-libvirt-opensuse-151 + +RUN zypper install -y \ + perl \ + perl-Archive-Tar \ + perl-CPAN-Changes \ + perl-Module-Build \ + perl-Test-Pod \ + perl-Test-Pod-Coverage \ + perl-XML-XPath && \ + zypper clean --all diff --git a/ci/libvirt-ubuntu-1604.dkr b/ci/libvirt-ubuntu-1604.dkr new file mode 100644 index 0000000..e1eb862 --- /dev/null +++ b/ci/libvirt-ubuntu-1604.dkr @@ -0,0 +1,14 @@ +FROM quay.io/libvirt/buildenv-libvirt-ubuntu-1604 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + libcpan-changes-perl \ + libmodule-build-perl \ + libtest-pod-coverage-perl \ + libtest-pod-perl \ + libtime-hr-perl \ + libxml-xpath-perl \ + perl && \ + apt-get autoremove -y && \ + apt-get autoclean -y diff --git a/ci/libvirt-ubuntu-1804.dkr b/ci/libvirt-ubuntu-1804.dkr new file mode 100644 index 0000000..b982dd2 --- /dev/null +++ b/ci/libvirt-ubuntu-1804.dkr @@ -0,0 +1,14 @@ +FROM quay.io/libvirt/buildenv-libvirt-ubuntu-1804 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + libcpan-changes-perl \ + libmodule-build-perl \ + libtest-pod-coverage-perl \ + libtest-pod-perl \ + libtime-hr-perl \ + libxml-xpath-perl \ + perl && \ + apt-get autoremove -y && \ + apt-get autoclean -y diff --git a/ci/refresh b/ci/refresh new file mode 100755 index 0000000..20476ee --- /dev/null +++ b/ci/refresh @@ -0,0 +1,16 @@ +#!/bin/sh + +if test -z "$1" +then + echo "syntax: $0 PATH-TO-LCITOOL" + exit 1 +fi + +LCITOOL=$1 + +HOSTS=$(${LCITOOL} hosts | grep -v freebsd) + +for host in ${HOSTS} +do + ${LCITOOL} dockerfile --inherit quay.io/libvirt/buildenv-$host $host libvirt-perl > $host.dkr +done -- 2.25.3