[libvirt PATCH v2 1/1] cirrus: Generate jobs dynamically

Instead of having static job definitions for FreeBSD and macOS, use a generic template for both and fill in the details that are actually different, such as the list of packages to install, in the GitLab CI job, right before calling cirrus-run. The target-specific information are provided by lcitool, so that keeping them up to date is just a matter of running the refresh script when necessary. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- .gitlab-ci.yml | 36 ++++++++++++++- ci/cirrus/build.yml | 26 +++++++++++ ci/cirrus/freebsd-12.yml.j2 | 73 ------------------------------- ci/cirrus/libvirt-freebsd-12.vars | 7 +++ ci/cirrus/libvirt-macos-1015.vars | 7 +++ ci/cirrus/macos-1015.yml.j2 | 38 ---------------- ci/cirrus/refresh | 22 ++++++++++ 7 files changed, 97 insertions(+), 112 deletions(-) create mode 100644 ci/cirrus/build.yml delete mode 100644 ci/cirrus/freebsd-12.yml.j2 create mode 100644 ci/cirrus/libvirt-freebsd-12.vars create mode 100644 ci/cirrus/libvirt-macos-1015.vars delete mode 100644 ci/cirrus/macos-1015.yml.j2 create mode 100755 ci/cirrus/refresh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e6eb2f9905..5565750b7e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,11 +58,35 @@ stages: # Jobs that we delegate to Cirrus CI because they require an operating # system other than Linux. These jobs will only run if the required # setup has been performed on the GitLab account (see ci/README.rst). +# +# The Cirrus CI configuration is generated by replacing target-specific +# variables in a generic template: some of these variables are provided +# when the GitLab CI job is defined, others are taken from a shell +# snippet generated using lcitool. +# +# Note that the $PATH environment variable has to be treated with +# special care, because we can't just override it at the GitLab CI job +# definition level or we risk breaking it completely. .cirrus_build_job_template: &cirrus_build_job_definition stage: builds image: registry.gitlab.com/libvirt/libvirt-ci/cirrus-run:master script: - - cirrus-run ci/cirrus/$NAME.yml.j2 + - source ci/cirrus/libvirt-$NAME.vars + - sed -e "s|[@]CI_REPOSITORY_URL@|$CI_REPOSITORY_URL|g" + -e "s|[@]CI_COMMIT_REF_NAME@|$CI_COMMIT_REF_NAME|g" + -e "s|[@]CI_COMMIT_SHA@|$CI_COMMIT_SHA|g" + -e "s|[@]CIRRUS_VM_INSTANCE_TYPE@|$CIRRUS_VM_INSTANCE_TYPE|g" + -e "s|[@]CIRRUS_VM_IMAGE_SELECTOR@|$CIRRUS_VM_IMAGE_SELECTOR|g" + -e "s|[@]CIRRUS_VM_IMAGE_NAME@|$CIRRUS_VM_IMAGE_NAME|g" + -e "s|[@]INSTALL_COMMAND@|$INSTALL_COMMAND|g" + -e "s|[@]PATH@|$PATH_EXTRA${PATH_EXTRA:+:}\$PATH|g" + -e "s|[@]PKG_CONFIG_PATH@|$PKG_CONFIG_PATH|g" + -e "s|[@]PKGS@|$PKGS|g" + -e "s|[@]MAKE@|$MAKE|g" + -e "s|[@]PYTHON@|$PYTHON|g" + <ci/cirrus/build.yml >ci/cirrus/$NAME.yml + - cat ci/cirrus/$NAME.yml + - cirrus-run ci/cirrus/$NAME.yml only: variables: - $CIRRUS_GITHUB_REPO @@ -351,11 +375,21 @@ x64-freebsd-12-build: <<: *cirrus_build_job_definition variables: NAME: freebsd-12 + CIRRUS_VM_INSTANCE_TYPE: freebsd_instance + CIRRUS_VM_IMAGE_SELECTOR: image_family + CIRRUS_VM_IMAGE_NAME: freebsd-12-1 + INSTALL_COMMAND: pkg install -y x64-macos-1015-build: <<: *cirrus_build_job_definition variables: NAME: macos-1015 + CIRRUS_VM_INSTANCE_TYPE: osx_instance + CIRRUS_VM_IMAGE_SELECTOR: image + CIRRUS_VM_IMAGE_NAME: catalina-base + INSTALL_COMMAND: brew install + PATH_EXTRA: /usr/local/opt/ccache/libexec:/usr/local/opt/gettext/bin:/usr/local/opt/libpcap/bin:/usr/local/opt/libxslt/bin:/usr/local/opt/rpcgen/bin + PKG_CONFIG_PATH: /usr/local/opt/curl/lib/pkgconfig:/usr/local/opt/libpcap/lib/pkgconfig:/usr/local/opt/libxml2/lib/pkgconfig:/usr/local/opt/ncurses/lib/pkgconfig:/usr/local/opt/readline/lib/pkgconfig # Cross compiled build jobs diff --git a/ci/cirrus/build.yml b/ci/cirrus/build.yml new file mode 100644 index 0000000000..893e13d724 --- /dev/null +++ b/ci/cirrus/build.yml @@ -0,0 +1,26 @@ +@CIRRUS_VM_INSTANCE_TYPE@: + @CIRRUS_VM_IMAGE_SELECTOR@: @CIRRUS_VM_IMAGE_NAME@ + +env: + CI_REPOSITORY_URL: "@CI_REPOSITORY_URL@" + CI_COMMIT_REF_NAME: "@CI_COMMIT_REF_NAME@" + CI_COMMIT_SHA: "@CI_COMMIT_SHA@" + PATH: "@PATH@" + PKG_CONFIG_PATH: "@PKG_CONFIG_PATH@" + PYTHON: "@PYTHON@" + MAKE: "@MAKE@" + +build_task: + install_script: + - @INSTALL_COMMAND@ @PKGS@ + clone_script: + - git clone --depth 100 "$CI_REPOSITORY_URL" . + - git fetch origin "$CI_COMMIT_REF_NAME" + - git reset --hard "$CI_COMMIT_SHA" + build_script: + - mkdir build + - cd build + - ../autogen.sh --prefix=$(pwd)/install-root + - $MAKE -j3 + - $MAKE -j3 install + - $MAKE -j3 dist diff --git a/ci/cirrus/freebsd-12.yml.j2 b/ci/cirrus/freebsd-12.yml.j2 deleted file mode 100644 index a653996850..0000000000 --- a/ci/cirrus/freebsd-12.yml.j2 +++ /dev/null @@ -1,73 +0,0 @@ -freebsd_instance: - image_family: freebsd-12-1 - -env: - CI_REPOSITORY_URL: {{ CI_REPOSITORY_URL }} - CI_COMMIT_REF_NAME: {{ CI_COMMIT_REF_NAME }} - CI_COMMIT_SHA: {{ CI_COMMIT_SHA }} - -freebsd_12_task: - install_script: - - pkg install -y - augeas - autoconf - automake - avahi - bash - bash-completion - ca_root_nss - ccache - chrony - cppi - curl - cyrus-sasl - dbus - diskscrub - dnsmasq - fusefs-libs - gdb - gettext - gettext-tools - git - glib - gmake - gnutls - hal - libpcap - libpciaccess - libssh - libssh2 - libtool - libxml2 - libxslt - lsof - meson - ncurses - ninja - patch - perl5 - pkgconf - polkit - py37-docutils - py37-flake8 - py37-setuptools - py37-wheel - python3 - qemu-utils - radvd - readline - screen - sudo - vim - yajl - clone_script: - - git clone --depth 100 "$CI_REPOSITORY_URL" . - - git fetch origin "$CI_COMMIT_REF_NAME" - - git reset --hard "$CI_COMMIT_SHA" - build_script: - - mkdir build - - cd build - - ../autogen.sh --prefix=$(pwd)/install-root - - gmake -j3 - - gmake -j3 install - - gmake -j3 dist diff --git a/ci/cirrus/libvirt-freebsd-12.vars b/ci/cirrus/libvirt-freebsd-12.vars new file mode 100644 index 0000000000..c7d4fbab0f --- /dev/null +++ b/ci/cirrus/libvirt-freebsd-12.vars @@ -0,0 +1,7 @@ +PACKAGING_COMMAND='pkg' +CC='/usr/bin/clang' +CCACHE='/usr/local/bin/ccache' +MAKE='/usr/local/bin/gmake' +NINJA='/usr/local/bin/ninja' +PYTHON='/usr/local/bin/python3' +PKGS='augeas autoconf automake avahi bash bash-completion ca_root_nss ccache chrony cppi curl cyrus-sasl dbus diskscrub dnsmasq fusefs-libs gdb gettext gettext-tools git glib gmake gnutls hal libpcap libpciaccess libssh libssh2 libtool libxml2 libxslt lsof meson ncurses ninja p5-App-cpanminus patch perl5 pkgconf polkit py37-docutils py37-flake8 py37-pip py37-setuptools py37-wheel python3 qemu-utils radvd readline screen sudo vim yajl' diff --git a/ci/cirrus/libvirt-macos-1015.vars b/ci/cirrus/libvirt-macos-1015.vars new file mode 100644 index 0000000000..a14c41a277 --- /dev/null +++ b/ci/cirrus/libvirt-macos-1015.vars @@ -0,0 +1,7 @@ +PACKAGING_COMMAND='brew' +CC='/usr/bin/clang' +CCACHE='/usr/local/bin/ccache' +MAKE='/usr/local/bin/gmake' +NINJA='/usr/local/bin/ninja' +PYTHON='/usr/local/bin/python3' +PKGS='augeas autoconf automake bash bash-completion ccache cpanminus cppi curl dbus dnsmasq docutils flake8 gdb gettext git glib gnutls gpatch libiscsi libpcap libssh libssh2 libtool libxml2 libxslt lsof make meson ncurses ninja perl pkg-config python3 qemu readline rpcgen screen scrub vim xz yajl' diff --git a/ci/cirrus/macos-1015.yml.j2 b/ci/cirrus/macos-1015.yml.j2 deleted file mode 100644 index d8aa9715d6..0000000000 --- a/ci/cirrus/macos-1015.yml.j2 +++ /dev/null @@ -1,38 +0,0 @@ -osx_instance: - image: catalina-base - -env: - CI_REPOSITORY_URL: {{ CI_REPOSITORY_URL }} - CI_COMMIT_REF_NAME: {{ CI_COMMIT_REF_NAME }} - CI_COMMIT_SHA: {{ CI_COMMIT_SHA }} - PATH: /usr/local/opt/gettext/bin:/usr/local/opt/ccache/libexec:/usr/local/opt/rpcgen/bin:$PATH - PKG_CONFIG_PATH: /usr/local/opt/libxml2/lib/pkgconfig - -macos_1015_task: - install_script: - - brew install - autoconf - automake - ccache - docutils - glib - gnutls - libtool - libxml2 - make - pkg-config - python - rpcgen - xz - yajl - clone_script: - - git clone --depth 100 "$CI_REPOSITORY_URL" . - - git fetch origin "$CI_COMMIT_REF_NAME" - - git reset --hard "$CI_COMMIT_SHA" - build_script: - - mkdir build - - cd build - - ../autogen.sh --prefix=$(pwd)/install-root - - gmake -j3 - - gmake -j3 install - - gmake -j3 dist diff --git a/ci/cirrus/refresh b/ci/cirrus/refresh new file mode 100755 index 0000000000..b84910a645 --- /dev/null +++ b/ci/cirrus/refresh @@ -0,0 +1,22 @@ +#!/bin/sh + +if test -z "$1" +then + echo "syntax: $0 PATH-TO-LCITOOL" + exit 1 +fi + +LCITOOL=$1 + +if ! test -x "$LCITOOL" +then + echo "$LCITOOL is not executable" + exit 1 +fi + +HOSTS=$($LCITOOL hosts | grep -E 'freebsd-12|macos') + +for host in $HOSTS +do + $LCITOOL dockerfile "$host" libvirt --variables >"$host.vars" +done -- 2.25.4