[libvirt PATCH v2 1/1] ci: Add helper script

This is intended to be the perform a number of CI-related operations that currently are implemented in various different scripts written in various different programming languages; in this first iteration it does two things: * implement the functionality of the existing "refresh" scripts, which it supersedes; * provide a nicer front-end for a subset of the functionality exposed by the ci/Makefile scaffolding, such as running basic builds. Over time, the plan is to rewrite all CI-related functionality in Python and move it into this script. Advantages: * it provides a more standard, more convenient command line interface; * it refreshes all lcitool-generated files in one go; * it can be called from the top-level source directory; * it automatically finds lcitool if it's somewhere in the user's $PATH; * it produces some output to keep the user updated on the progress of the current operation; * it's written in a real programming language, which will hopefully help maintainability. Signed-off-by: Andrea Bolognani <abologna(a)redhat.com&gt; --- ci/cirrus/refresh | 22 ----- ci/containers/refresh | 41 --------- ci/helper | 187 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 63 deletions(-) delete mode 100755 ci/cirrus/refresh delete mode 100755 ci/containers/refresh create mode 100755 ci/helper diff --git a/ci/cirrus/refresh b/ci/cirrus/refresh deleted file mode 100755 index 63ca794134..0000000000 --- a/ci/cirrus/refresh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -if test -z "$1" -then - echo "syntax: $0 PATH-TO-LCITOOL" - exit 1 -fi - -LCITOOL=$1 - -if ! test -x "$LCITOOL" -then - echo "$LCITOOL is not executable" - exit 1 -fi - -HOSTS=$($LCITOOL hosts | grep -E 'freebsd|macos') - -for host in $HOSTS -do - $LCITOOL variables "$host" libvirt >"$host.vars" -done diff --git a/ci/containers/refresh b/ci/containers/refresh deleted file mode 100755 index f38d3634b5..0000000000 --- a/ci/containers/refresh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh - -if test -z "$1" -then - echo "syntax: $0 PATH-TO-LCITOOL" - exit 1 -fi - -LCITOOL=$1 - -if ! test -x "$LCITOOL" -then - echo "$LCITOOL is not executable" - exit 1 -fi - -HOSTS=$($LCITOOL hosts | grep -Ev 'freebsd|macos') - -for host in $HOSTS -do - case "$host" in - fedora-rawhide) - for cross in mingw32 mingw64 - do - $LCITOOL dockerfile $host libvirt --cross $cross > ci-$host-cross-$cross.Dockerfile - done - ;; - debian-*) - for cross in aarch64 armv6l armv7l i686 mips mips64el mipsel ppc64le s390x - do - if test "$host-cross-$cross" = "debian-sid-cross-mips" - then - continue - fi - $LCITOOL dockerfile $host libvirt --cross $cross > ci-$host-cross-$cross.Dockerfile - done - ;; - esac - - $LCITOOL dockerfile $host libvirt > ci-$host.Dockerfile -done diff --git a/ci/helper b/ci/helper new file mode 100755 index 0000000000..dec24ac741 --- /dev/null +++ b/ci/helper @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see +# <http://www.gnu.org/licenses/>;. + +import argparse +import os +import pty +import shutil +import subprocess +import sys + + +class Parser: + def __init__(self): + self.parser = argparse.ArgumentParser() + subparsers = self.parser.add_subparsers( + dest="action", + metavar="ACTION", + ) + subparsers.required = True + + buildparser = subparsers.add_parser( + "build", + help="run a build in a container", + ) + self.add_target_arg(buildparser) + self.add_engine_arg(buildparser) + self.add_login_arg(buildparser) + + shellparser = subparsers.add_parser( + "shell", + help="start a shell in a container", + ) + self.add_target_arg(shellparser) + self.add_engine_arg(shellparser) + self.add_login_arg(shellparser) + + refreshparser = subparsers.add_parser( + "refresh", + help="refresh data generated with lcitool", + ) + self.add_lcitool_arg(refreshparser) + + def add_target_arg(self, parser): + parser.add_argument( + "target", + help="build on target OS", + ) + + def add_engine_arg(self, parser): + parser.add_argument( + "--engine", + choices=["auto", "podman", "docker"], + default="auto", + help="container engine to use", + ) + + def add_login_arg(self, parser): + parser.add_argument( + "--login", + default=os.getlogin(), + help="login to use inside the container", + ) + + def add_lcitool_arg(self, parser): + parser.add_argument( + "--lcitool", + metavar="PATH", + default="lcitool", + help="path to lcitool binary", + ) + + def parse(self): + return self.parser.parse_args() + + +class Application: + def __init__(self): + self.basedir = os.path.dirname(os.path.realpath(__file__)) + + args = Parser().parse() + self.action = args.action + + if args.action == "refresh": + self.lcitool = args.lcitool + if not shutil.which(self.lcitool): + sys.exit("error: 'lcitool' not installed") + + elif args.action in ["build", "shell"]: + self.target = args.target + self.engine = args.engine + self.login = args.login + + def make_run(self, target): + args = [ + "-C", self.basedir, target, + f"CI_ENGINE={self.engine}", + f"CI_USER_LOGIN={self.login}", + ] + + if pty.spawn(["make"] + args) != 0: + raise Exception("make failed") + + def lcitool_run(self, args): + output = subprocess.check_output([self.lcitool] + args) + return output.decode("utf-8") + + def lcitool_get_hosts(self): + output = self.lcitool_run(["hosts"]) + return output.splitlines() + + def generate_dockerfile(self, host, cross=None): + args = ["dockerfile", host, "libvirt"] + outfile = f"{self.basedir}/containers/ci-{host}.Dockerfile" + + if cross: + args.extend(["--cross", cross]) + outfile = f"{self.basedir}/containers/ci-{host}-cross-{cross}.Dockerfile" + + output = self.lcitool_run(args) + with open(outfile, "w") as f: + f.write(output) + + def generate_vars(self, host): + output = self.lcitool_run(["variables", host, "libvirt"]) + with open(f"{self.basedir}/cirrus/{host}.vars", "w") as f: + f.write(output) + + def refresh_containers(self): + for host in self.lcitool_get_hosts(): + if "freebsd" in host or "macos" in host: + continue + + if host == "fedora-rawhide": + for cross in ["mingw32", "mingw64"]: + print(f"containers/{host} ({cross})") + self.generate_dockerfile(host, cross) + + if "debian-" in host: + for cross in ["aarch64", "armv6l", "armv7l", "i686", "mips", "mips64el", "mipsel", "ppc64le", "s390x"]: + if host == "debian-sid" and cross == "mips": + continue + print(f"containers/{host} ({cross})") + self.generate_dockerfile(host, cross) + + print(f"containers/{host}") + self.generate_dockerfile(host) + + def refresh_cirrus(self): + for host in self.lcitool_get_hosts(): + if "freebsd" not in host and "macos" not in host: + continue + + print(f"cirrus/{host}") + self.generate_vars(host) + + def action_refresh(self): + self.refresh_containers() + self.refresh_cirrus() + + def action_build(self): + self.make_run(f&quot;ci-build(a){self.target}&quot;) + + def action_shell(self): + self.make_run(f&quot;ci-shell(a){self.target}&quot;) + + def run(self): + method = "action_{}".format(self.action.replace("-", "_")) + getattr(self, method).__call__() + + +if __name__ == "__main__": + Application().run() -- 2.26.2