We're going to need it in a bit. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- build-aux/syntax-check.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/build-aux/syntax-check.mk b/build-aux/syntax-check.mk index 794ec326e4..c5a1b05f60 100644 --- a/build-aux/syntax-check.mk +++ b/build-aux/syntax-check.mk @@ -497,6 +497,7 @@ sc_prohibit_PATH_MAX: include $(top_srcdir)/build-aux/Makefile.nonreentrant sc_prohibit_nonreentrant: @prohibit="\\<(${NON_REENTRANT_RE}) *\\(" \ + exclude='exempt from syntax-check' \ halt="use re-entrant functions (usually ending with _r)" \ $(_sc_search_regexp) -- 2.26.2

This is intended to be perform a number of CI-related operations that are currently implemented in various different scripts written in various different programming languages. Eventually, all existing functionality will be reimplemented in Python and made available through this single entry point; for now, let's start with a very basic skeleton. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ci/helper | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 ci/helper diff --git a/ci/helper b/ci/helper new file mode 100755 index 0000000000..2a59b8e5ab --- /dev/null +++ b/ci/helper @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>;. + +import argparse +import pathlib + + +class Parser: + def __init__(self): + # Main parser + self.parser = argparse.ArgumentParser() + subparsers = self.parser.add_subparsers( + dest="action", + metavar="ACTION", + ) + subparsers.required = True + + def parse(self): + return self.parser.parse_args() + + +class Application: + def __init__(self): + self.basedir = pathlib.Path(__file__).resolve().parent + self.args = Parser().parse() + + def run(self): + self.args.func(self) + + +if __name__ == "__main__": + Application().run() -- 2.26.2

This provides the same functionality as the two refresh scripts that are currently in the repository, with the following advantages: * all files are refreshed with a single command; * if lcitool is present in the user's $PATH, it will be discovered and used automatically; * some output is produced, so the user can follow along with the progress of the operation. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ci/helper | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/ci/helper b/ci/helper index 2a59b8e5ab..05ecdcb426 100755 --- a/ci/helper +++ b/ci/helper @@ -18,10 +18,22 @@ import argparse import pathlib +import shutil +import subprocess +import sys class Parser: def __init__(self): + # Options that are common to all actions that use lcitool + lcitoolparser = argparse.ArgumentParser(add_help=False) + lcitoolparser.add_argument( + "--lcitool", + metavar="PATH", + default="lcitool", + help="path to lcitool binary", + ) + # Main parser self.parser = argparse.ArgumentParser() subparsers = self.parser.add_subparsers( @@ -30,6 +42,14 @@ class Parser: ) subparsers.required = True + # refresh action + refreshparser = subparsers.add_parser( + "refresh", + help="refresh data generated with lcitool", + parents=[lcitoolparser], + ) + refreshparser.set_defaults(func=Application.action_refresh) + def parse(self): return self.parser.parse_args() @@ -39,6 +59,88 @@ class Application: self.basedir = pathlib.Path(__file__).resolve().parent self.args = Parser().parse() + if self.args.action == "refresh": + if not shutil.which(self.args.lcitool): + sys.exit("error: 'lcitool' not installed") + + def lcitool_run(self, args): + output = subprocess.check_output([self.args.lcitool] + args) + return output.decode("utf-8") + + def lcitool_get_hosts(self): + output = self.lcitool_run(["hosts"]) + return output.splitlines() + + def generate_dockerfile(self, host, cross=None): + args = ["dockerfile", host, "libvirt"] + outdir = self.basedir.joinpath("containers") + outfile = f"ci-{host}.Dockerfile" + + if cross: + args.extend(["--cross", cross]) + outfile = f"ci-{host}-cross-{cross}.Dockerfile" + + output = self.lcitool_run(args) + with open(outdir.joinpath(outfile), "w") as f: + f.write(output) + + def generate_vars(self, host): + args = ["variables", host, "libvirt"] + outdir = self.basedir.joinpath("cirrus") + outfile = f"{host}.vars" + + output = self.lcitool_run(args) + with open(outdir.joinpath(outfile), "w") as f: + f.write(output) + + def refresh_containers(self): + debian_cross = [ + "aarch64", + "armv6l", + "armv7l", + "i686", + "mips", + "mips64el", + "mipsel", + "ppc64le", + "s390x", + ] + fedora_cross = [ + "mingw32", + "mingw64", + ] + + for host in self.lcitool_get_hosts(): + if host.startswith("freebsd-") or host.startswith("macos-"): + continue + + print(f"containers/{host}") + self.generate_dockerfile(host) + + if host == "fedora-rawhide": + for cross in fedora_cross: + print(f"containers/{host} ({cross})") + self.generate_dockerfile(host, cross) + + if host.startswith("debian-"): + for cross in debian_cross: + if host == "debian-sid" and cross == "mips": + continue + print(f"containers/{host} ({cross})") + self.generate_dockerfile(host, cross) + + def refresh_cirrus(self): + for host in self.lcitool_get_hosts(): + if not (host.startswith("freebsd-") or host.startswith("macos-")): + continue + + print(f"cirrus/{host}") + self.generate_vars(host) + + def action_refresh(self): + self.refresh_containers() + self.refresh_cirrus() + def run(self): self.args.func(self) -- 2.26.2

This simply calls the underlying Makefile target. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ci/helper | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ci/helper b/ci/helper index 05ecdcb426..420e9b73c2 100755 --- a/ci/helper +++ b/ci/helper @@ -18,6 +18,7 @@ import argparse import pathlib +import pty import shutil import subprocess import sys @@ -42,6 +43,13 @@ class Parser: ) subparsers.required = True + # list-images action + listimagesparser = subparsers.add_parser( + "list-images", + help="list known container images", + ) + listimagesparser.set_defaults(func=Application.action_list_images) + # refresh action refreshparser = subparsers.add_parser( "refresh", @@ -63,6 +71,16 @@ class Application: if not shutil.which(self.args.lcitool): sys.exit("error: 'lcitool' not installed") + def make_run(self, target): + args = [ + "-C", + self.basedir, + target, + ] + + if pty.spawn(["make"] + args) != 0: + sys.exit("error: 'make' failed") + def lcitool_run(self, args): output = subprocess.check_output([self.args.lcitool] + args) return output.decode("utf-8") @@ -137,6 +155,9 @@ class Application: print(f"cirrus/{host}") self.generate_vars(host) + def action_list_images(self): + self.make_run(f"ci-list-images") + def action_refresh(self): self.refresh_containers() self.refresh_cirrus() -- 2.26.2

On Fri, Mar 12, 2021 at 06:28:18PM +0100, Andrea Bolognani wrote: With the 'shell' action, not sure if "build" is the right word in that case, how about "perform action on target OS" -- since the "target" argument is shared among ['shell', 'build', 'test']. Reviewed-by: Erik Skultety <eskultet(a)redhat.com&gt;

This simply calls the underlying Makefile target, but allows additional arguments to be specified in a more convenient and discoverable way. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ci/helper | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/ci/helper b/ci/helper index 8eb521ae40..4a552595df 100755 --- a/ci/helper +++ b/ci/helper @@ -55,6 +55,20 @@ class Parser: help="use container images with non-default tags", ) + # Options that are common to all actions that call the + # project's build system + mesonparser = argparse.ArgumentParser(add_help=False) + mesonparser.add_argument( + "--meson-args", + default="", + help="additional arguments passed to meson", + ) + mesonparser.add_argument( + "--ninja-args", + default="", + help="additional arguments passed to ninja", + ) + # Options that are common to all actions that use lcitool lcitoolparser = argparse.ArgumentParser(add_help=False) lcitoolparser.add_argument( @@ -72,6 +86,14 @@ class Parser: ) subparsers.required = True + # build action + buildparser = subparsers.add_parser( + "build", + help="run a build in a container", + parents=[containerparser, mesonparser], + ) + buildparser.set_defaults(func=Application.action_build) + # shell action shellparser = subparsers.add_parser( "shell", @@ -115,7 +137,7 @@ class Application: target, ] - if self.args.action == "shell": + if self.args.action in ["build", "shell"]: args.extend([ f"CI_ENGINE={self.args.engine}", f"CI_USER_LOGIN={self.args.login}", @@ -123,6 +145,12 @@ class Application: f"CI_IMAGE_TAG={self.args.image_tag}", ]) + if self.args.action == "build": + args.extend([ + f"CI_MESON_ARGS={self.args.meson_args}", + f"CI_NINJA_ARGS={self.args.ninja_args}", + ]) + if pty.spawn(["make"] + args) != 0: sys.exit("error: 'make' failed") @@ -200,6 +228,9 @@ class Application: print(f"cirrus/{host}") self.generate_vars(host) + def action_build(self): + self.make_run(f&quot;ci-build(a){self.args.target}&quot;) + def action_shell(self): self.make_run(f&quot;ci-shell(a){self.args.target}&quot;) -- 2.26.2

This simply calls the underlying Makefile target, but allows additional arguments to be specified in a more convenient and discoverable way. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ci/helper | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ci/helper b/ci/helper index 4a552595df..4d6dfab19b 100755 --- a/ci/helper +++ b/ci/helper @@ -94,6 +94,14 @@ class Parser: ) buildparser.set_defaults(func=Application.action_build) + # test action + testparser = subparsers.add_parser( + "test", + help="run a build in a container (including tests)", + parents=[containerparser, mesonparser], + ) + testparser.set_defaults(func=Application.action_test) + # shell action shellparser = subparsers.add_parser( "shell", @@ -137,7 +145,7 @@ class Application: target, ] - if self.args.action in ["build", "shell"]: + if self.args.action in ["build", "test", "shell"]: args.extend([ f"CI_ENGINE={self.args.engine}", f"CI_USER_LOGIN={self.args.login}", @@ -145,7 +153,7 @@ class Application: f"CI_IMAGE_TAG={self.args.image_tag}", ]) - if self.args.action == "build": + if self.args.action in ["build", "test"]: args.extend([ f"CI_MESON_ARGS={self.args.meson_args}", f"CI_NINJA_ARGS={self.args.ninja_args}", @@ -231,6 +239,9 @@ class Application: def action_build(self): self.make_run(f&quot;ci-build(a){self.args.target}&quot;) + def action_test(self): + self.make_run(f&quot;ci-test(a){self.args.target}&quot;) + def action_shell(self): self.make_run(f&quot;ci-shell(a){self.args.target}&quot;) -- 2.26.2

We now wrap all its important functionality with the much more user-friendly ci/helper script, and the long term plan is for the Makefile to disappear completely. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ci/Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/Makefile b/ci/Makefile index fc0d2590db..72f5bda942 100644 --- a/ci/Makefile +++ b/ci/Makefile @@ -223,6 +223,17 @@ ci-list-images: @echo ci-help: + @echo + @echo + @echo + @echo " !!! PLEASE DON'T USE THIS DIRECTLY !!!" + @echo + @echo " Use the ci/helper script instead" + @echo + @echo " !!! PLEASE DON'T USE THIS DIRECTLY !!!" + @echo + @echo + @echo @echo "Build libvirt inside containers used for CI" @echo @echo "Available targets:" -- 2.26.2