We only use the list of submodules once, so no need to store it in a variable. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- Makefile.ci | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Makefile.ci b/Makefile.ci index 14d595a00f..86e936aef8 100644 --- a/Makefile.ci +++ b/Makefile.ci @@ -42,10 +42,6 @@ CI_MAKE_ARGS = # Any extra arguments to pass to configure CI_CONFIGURE_ARGS = -# Avoid pulling submodules over the network by locally -# cloning them -CI_SUBMODULES = $(shell git submodule | awk '{ print $$2 }') - # Location of the container images we're going to pull # Can be useful to overridde to use a locally built # image instead @@ -182,7 +178,7 @@ ci-prepare-tree: ci-check-engine cp /etc/group $(CI_SCRATCHDIR); \ echo "Cloning $(CI_GIT_ROOT) to $(CI_HOST_SRCDIR)"; \ git clone $(CI_GIT_ARGS) $(CI_GIT_ROOT) $(CI_HOST_SRCDIR) || exit 1; \ - for mod in $(CI_SUBMODULES) ; \ + for mod in $$(git submodule | awk '{ print $$2 }') ; \ do \ test -f $(CI_GIT_ROOT)/$$mod/.git || continue ; \ echo "Cloning $(CI_GIT_ROOT)/$$mod to $(CI_HOST_SRCDIR)/$$mod"; \ -- 2.21.0

Some applications expect the user's home directory to be present on the system and require workarounds when that's not the case. Creating the home directory along with everything else is easy enough for us, so let's just do that. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ci/Makefile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ci/Makefile b/ci/Makefile index 350eb636cd..f52a0bf621 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -65,6 +65,11 @@ CI_REUSE = 0 CI_UID = $(shell id -u) CI_GID = $(shell id -g) +# We also need the user's login and home directory to prepare the +# environment the way some programs expect it +CI_USER_LOGIN = $(shell echo "$$USER") +CI_USER_HOME = $(shell echo "$$HOME") + CI_ENGINE = auto # Container engine we are going to use, can be overridden per make # invocation, if it is not we try podman and then default to docker. @@ -87,6 +92,10 @@ CI_PWDB_MOUNTS = \ --volume $(CI_SCRATCHDIR)/passwd:/etc/passwd:ro,z \ $(NULL) +CI_HOME_MOUNTS = \ + --volume $(CI_SCRATCHDIR)/home:$(CI_USER_HOME):z \ + $(NULL) + # Docker containers can have very large ulimits # for nofiles - as much as 1048576. This makes # libvirt very slow at exec'ing programs. @@ -109,8 +118,8 @@ ifeq ($(CI_ENGINE),podman) # need to be higher, but that only happens when your /etc/sub{u,g}id allow # users to have more IDs. Unless --keep-uid is supported, let's do this in a # way that should work for everyone. - CI_MAX_UID = $(shell sed -n "s/^$$USER:[^:]\+://p" /etc/subuid) - CI_MAX_GID = $(shell sed -n "s/^$$USER:[^:]\+://p" /etc/subgid) + CI_MAX_UID = $(shell sed -n "s/^$(CI_USER_LOGIN):[^:]\+://p" /etc/subuid) + CI_MAX_GID = $(shell sed -n "s/^$(CI_USER_LOGIN):[^:]\+://p" /etc/subgid) ifeq ($(CI_MAX_UID),) CI_MAX_UID = 65536 endif @@ -159,6 +168,7 @@ CI_ENGINE_ARGS = \ --tty \ $(CI_PODMAN_ARGS) \ $(CI_PWDB_MOUNTS) \ + $(CI_HOME_MOUNTS) \ --volume $(CI_HOST_SRCDIR):$(CI_CONT_SRCDIR):z \ --workdir $(CI_CONT_SRCDIR) \ --ulimit nofile=$(CI_ULIMIT_FILES):$(CI_ULIMIT_FILES) \ @@ -175,6 +185,7 @@ ci-prepare-tree: ci-check-engine mkdir -p $(CI_SCRATCHDIR); \ cp /etc/passwd $(CI_SCRATCHDIR); \ cp /etc/group $(CI_SCRATCHDIR); \ + mkdir -p $(CI_SCRATCHDIR)/home; \ echo "Cloning $(CI_GIT_ROOT) to $(CI_HOST_SRCDIR)"; \ git clone $(CI_GIT_ARGS) $(CI_GIT_ROOT) $(CI_HOST_SRCDIR) || exit 1; \ for mod in $$(git submodule | awk '{ print $$2 }' | sed -E 's,^../,,g') ; \ -- 2.21.0

Instead of hardcoding build instructions into the Makefile, move them to a separate script that's mounted into the container. This gives us a couple of advantages: we no longer have to deal with the awkward quoting required when embedding shell code in a Makefile, and we also provide the users with a way to override the default build instructions with their own. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- Makefile.am | 1 + ci/Makefile | 48 ++++++++++++++++++------------------------------ ci/build.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 30 deletions(-) create mode 100644 ci/build.sh diff --git a/Makefile.am b/Makefile.am index b743b4b08b..96fac92186 100644 --- a/Makefile.am +++ b/Makefile.am @@ -51,6 +51,7 @@ EXTRA_DIST = \ build-aux/useless-if-before-free \ build-aux/vc-list-files \ ci/Makefile \ + ci/build.sh \ $(NULL) pkgconfigdir = $(libdir)/pkgconfig diff --git a/ci/Makefile b/ci/Makefile index dc7ee6c037..f7e31b4f97 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -41,6 +41,9 @@ CI_MAKE_ARGS = # Any extra arguments to pass to configure CI_CONFIGURE_ARGS = +# Script containing build instructions +CI_BUILD_SCRIPT = $(CI_ROOTDIR)/build.sh + # Location of the container images we're going to pull # Can be useful to overridde to use a locally built # image instead @@ -96,6 +99,10 @@ CI_HOME_MOUNTS = \ --volume $(CI_SCRATCHDIR)/home:$(CI_USER_HOME):z \ $(NULL) +CI_SCRIPT_MOUNTS = \ + --volume $(CI_SCRATCHDIR)/build:$(CI_USER_HOME)/build:z \ + $(NULL) + # Docker containers can have very large ulimits # for nofiles - as much as 1048576. This makes # libvirt very slow at exec'ing programs. @@ -169,6 +176,7 @@ CI_ENGINE_ARGS = \ $(CI_PODMAN_ARGS) \ $(CI_PWDB_MOUNTS) \ $(CI_HOME_MOUNTS) \ + $(CI_SCRIPT_MOUNTS) \ --volume $(CI_HOST_SRCDIR):$(CI_CONT_SRCDIR):z \ --workdir $(CI_CONT_SRCDIR) \ --ulimit nofile=$(CI_ULIMIT_FILES):$(CI_ULIMIT_FILES) \ @@ -186,6 +194,8 @@ ci-prepare-tree: ci-check-engine cp /etc/passwd $(CI_SCRATCHDIR); \ cp /etc/group $(CI_SCRATCHDIR); \ mkdir -p $(CI_SCRATCHDIR)/home; \ + cp "$(CI_BUILD_SCRIPT)" $(CI_SCRATCHDIR)/build; \ + chmod +x "$(CI_SCRATCHDIR)/build"; \ echo "Cloning $(CI_GIT_ROOT) to $(CI_HOST_SRCDIR)"; \ git clone $(CI_GIT_ARGS) $(CI_GIT_ROOT) $(CI_HOST_SRCDIR) || exit 1; \ for mod in $$(git submodule | awk '{ print $$2 }' | sed -E 's,^../,,g') ; \ @@ -196,38 +206,16 @@ ci-prepare-tree: ci-check-engine done ; \ fi -# $CONFIGURE_OPTS is a env that can optionally be set in the container, -# populated at build time from the Dockerfile. A typical use case would -# be to pass --host/--target args to trigger cross-compilation -# -# This can be augmented by make local args in $(CI_CONFIGURE_ARGS) -# -# gl_public_submodule_commit= to disable gnulib's submodule check -# which breaks due to way we clone the submodules ci-build@%: ci-prepare-tree $(CI_ENGINE) run $(CI_ENGINE_ARGS) $(CI_IMAGE_PREFIX)$*$(CI_IMAGE_TAG) \ - /bin/bash -c '\ - mkdir -p $(CI_CONT_BUILDDIR) || exit 1 ; \ - cd $(CI_CONT_BUILDDIR) ; \ - NOCONFIGURE=1 $(CI_CONT_SRCDIR)/autogen.sh || exit 1 ; \ - $(CI_CONFIGURE) $${CONFIGURE_OPTS} $(CI_CONFIGURE_ARGS) ; \ - if test $$? != 0 ; \ - then \ - test -f config.log && cat config.log ; \ - exit 1 ; \ - fi; \ - find -name test-suite.log -delete ; \ - export VIR_TEST_DEBUG=1 ; \ - make -j$(CI_SMP) gl_public_submodule_commit= $(CI_MAKE_ARGS) ; \ - if test $$? != 0 ; then \ - LOGS=`find -name test-suite.log` ; \ - if test "$${LOGS}" != "" ; then \ - echo "=== LOG FILE(S) START ===" ; \ - cat $${LOGS} ; \ - echo "=== LOG FILE(S) END ===" ; \ - fi ; \ - exit 1 ;\ - fi' + /bin/bash -c ' \ + export CI_CONT_SRCDIR="$(CI_CONT_SRCDIR)"; \ + export CI_CONT_BUILDDIR="$(CI_CONT_BUILDDIR)"; \ + export CI_SMP="$(CI_SMP)"; \ + export CI_CONFIGURE="$(CI_CONFIGURE)"; \ + export CI_CONFIGURE_ARGS="$(CI_CONFIGURE_ARGS)"; \ + export CI_MAKE_ARGS="$(CI_MAKE_ARGS)"; \ + $(CI_USER_HOME)/build || exit 1' @test "$(CI_CLEAN)" = "1" && rm -rf $(CI_SCRATCHDIR) || : ci-check@%: diff --git a/ci/build.sh b/ci/build.sh new file mode 100644 index 0000000000..0874c2d1d9 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,40 @@ +# This script is used to build libvirt inside the container. +# +# You can customize it to your liking, or alternatively use a +# completely different script by passing +# +# CI_BUILD_SCRIPT=/path/to/your/build/script +# +# to make. + +mkdir -p "$CI_CONT_BUILDDIR" || exit 1 +cd "$CI_CONT_BUILDDIR" + +export VIR_TEST_DEBUG=1 +NOCONFIGURE=1 "$CI_CONT_SRCDIR/autogen.sh" || exit 1 + +# $CONFIGURE_OPTS is a env that can optionally be set in the container, +# populated at build time from the Dockerfile. A typical use case would +# be to pass --host/--target args to trigger cross-compilation +# +# This can be augmented by make local args in $CI_CONFIGURE_ARGS +"$CI_CONFIGURE" $CONFIGURE_OPTS $CI_CONFIGURE_ARGS +if test $? != 0; then + test -f config.log && cat config.log + exit 1 +fi +find -name test-suite.log -delete + +# gl_public_submodule_commit= to disable gnulib's submodule check +# which breaks due to way we clone the submodules +make -j"$CI_SMP" gl_public_submodule_commit= $CI_MAKE_ARGS + +if test $? != 0; then \ + LOGS=$(find -name test-suite.log) + if test "$LOGS"; then + echo "=== LOG FILE(S) START ===" + cat $LOGS + echo "=== LOG FILE(S) END ===" + fi + exit 1 +fi -- 2.21.0

This script is run before $(CI_BUILD_SCRIPT) and can be used to tweak the environment as necessary before the build starts. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- Makefile.am | 1 + ci/Makefile | 8 +++++++- ci/prepare.sh | 9 +++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 ci/prepare.sh diff --git a/Makefile.am b/Makefile.am index 96fac92186..cf9ff94f4f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -52,6 +52,7 @@ EXTRA_DIST = \ build-aux/vc-list-files \ ci/Makefile \ ci/build.sh \ + ci/prepare.sh \ $(NULL) pkgconfigdir = $(libdir)/pkgconfig diff --git a/ci/Makefile b/ci/Makefile index 2b5be97238..17d85d407f 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -41,6 +41,9 @@ CI_MAKE_ARGS = # Any extra arguments to pass to configure CI_CONFIGURE_ARGS = +# Script containing environment preparation steps +CI_PREPARE_SCRIPT = $(CI_ROOTDIR)/prepare.sh + # Script containing build instructions CI_BUILD_SCRIPT = $(CI_ROOTDIR)/build.sh @@ -100,6 +103,7 @@ CI_HOME_MOUNTS = \ $(NULL) CI_SCRIPT_MOUNTS = \ + --volume $(CI_SCRATCHDIR)/prepare:$(CI_USER_HOME)/prepare:z \ --volume $(CI_SCRATCHDIR)/build:$(CI_USER_HOME)/build:z \ $(NULL) @@ -194,8 +198,9 @@ ci-prepare-tree: ci-check-engine cp /etc/passwd $(CI_SCRATCHDIR); \ cp /etc/group $(CI_SCRATCHDIR); \ mkdir -p $(CI_SCRATCHDIR)/home; \ + cp "$(CI_PREPARE_SCRIPT)" $(CI_SCRATCHDIR)/prepare; \ cp "$(CI_BUILD_SCRIPT)" $(CI_SCRATCHDIR)/build; \ - chmod +x "$(CI_SCRATCHDIR)/build"; \ + chmod +x "$(CI_SCRATCHDIR)/prepare" "$(CI_SCRATCHDIR)/build"; \ echo "Cloning $(CI_GIT_ROOT) to $(CI_HOST_SRCDIR)"; \ git clone $(CI_GIT_ARGS) $(CI_GIT_ROOT) $(CI_HOST_SRCDIR) || exit 1; \ for mod in $$(git submodule | awk '{ print $$2 }' | sed -E 's,^../,,g') ; \ @@ -209,6 +214,7 @@ ci-prepare-tree: ci-check-engine ci-run-command@%: ci-prepare-tree $(CI_ENGINE) run $(CI_ENGINE_ARGS) $(CI_IMAGE_PREFIX)$*$(CI_IMAGE_TAG) \ /bin/bash -c ' \ + $(CI_USER_HOME)/prepare || exit 1; \ export CI_CONT_SRCDIR="$(CI_CONT_SRCDIR)"; \ export CI_CONT_BUILDDIR="$(CI_CONT_BUILDDIR)"; \ export CI_SMP="$(CI_SMP)"; \ diff --git a/ci/prepare.sh b/ci/prepare.sh new file mode 100644 index 0000000000..f70107bd62 --- /dev/null +++ b/ci/prepare.sh @@ -0,0 +1,9 @@ +# This script is used to prepare the environment that will be used +# to build libvirt inside the container. +# +# You can customize it to your liking, or alternatively use a +# completely different script by passing +# +# CI_PREPARE_SCRIPT=/path/to/your/prepare/script +# +# to make. -- 2.21.0

Now that we're using sudo, the initial work directory is no longer relevant since the user will find themselves in their home directory when they get control anyway. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ci/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/Makefile b/ci/Makefile index 86a07049ac..35a35d6082 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -168,7 +168,6 @@ CI_GIT_ARGS = \ # as dev so that file ownership matches host # instead of root:root # --volume to pass in the cloned git repo & config -# --workdir to set cwd to vpath build location # --ulimit lower files limit for performance reasons # --interactive # --tty Ensure we have ability to Ctrl-C the build @@ -181,7 +180,6 @@ CI_ENGINE_ARGS = \ $(CI_HOME_MOUNTS) \ $(CI_SCRIPT_MOUNTS) \ --volume $(CI_HOST_SRCDIR):$(CI_CONT_SRCDIR):z \ - --workdir $(CI_CONT_SRCDIR) \ --ulimit nofile=$(CI_ULIMIT_FILES):$(CI_ULIMIT_FILES) \ --cap-add=SYS_PTRACE \ $(NULL) -- 2.21.0