[jenkins-ci PATCH 0/8] lcitool: Support MinGW cross-build Dockerfiles
More details in the commit message for patch 7/8. Pavel pointed out today that the current method of triggering MinGW builds using our CI scaffolding, eg. $ make ci-build@fedora-30 CI_CONFIGURE=mingw64-configure is easy to get wrong and not very discoverable, so I took that as motivation to implement a change that I had been thinking about for a long time anyway. The new usage will be $ make ci-build@fedora-30-cross-mingw64 which aligns with how we're already doing cross-builds for other architectures and is discoverable via 'make ci-list-images'. The implementation is not the prettiest, but the Dockerfile generator in general could use some love so I don't think this improvement should be blocked because of that; I'll try to spend some time refactoring and cleaning up once this has been merged. Andrea Bolognani (8): lcitool: Introduce cross_arch local variable lcitool: Change check for pip_pkgs formatting lcitool: Separate computation and formatting lcitool: Introduce _dockerfile_format() lcitool: Introduce _dockerfile_build_varmap() lcitool: Add RPM-specific _dockerfile_build_varmap() variant lcitool: Support MinGW cross-build Dockerfiles on Fedora lcitool: Add more checks to _action_dockerfile() guests/lcitool | 219 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 172 insertions(+), 47 deletions(-) -- 2.24.1
Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- guests/lcitool | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/guests/lcitool b/guests/lcitool index 8a681e4..6624887 100755 --- a/guests/lcitool +++ b/guests/lcitool @@ -657,15 +657,16 @@ class Application: os_name = facts["os_name"] os_version = facts["os_version"] os_full = os_name + os_version + cross_arch = args.cross_arch if package_format not in ["deb", "rpm"]: raise Exception("Host {} doesn't support Dockerfiles".format(host)) - if args.cross_arch: + if cross_arch: if os_name != "Debian": raise Exception("Cannot cross compile on {}".format(os_name)) - if args.cross_arch == self._native_arch: + if cross_arch == self._native_arch: raise Exception("Cross arch {} should differ from native {}". - format(args.cross_arch, self._native_arch)) + format(cross_arch, self._native_arch)) projects = self._projects.expand_pattern(args.projects) for project in projects: @@ -682,8 +683,8 @@ class Application: pip_pkgs = {} base_keys = ["default", package_format, os_name, os_full] cross_keys = [] - if args.cross_arch: - keys = base_keys + [args.cross_arch + "-" + k for k in base_keys] + if cross_arch: + keys = base_keys + [cross_arch + "-" + k for k in base_keys] cross_keys = ["cross-policy-" + k for k in base_keys] else: keys = base_keys + [self._native_arch + "-" + k for k in base_keys] @@ -727,10 +728,10 @@ class Application: varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) if package_format == "deb": - if args.cross_arch: - deb_arch = Util.native_arch_to_deb_arch(args.cross_arch) - abi = Util.native_arch_to_abi(args.cross_arch) - lib = Util.native_arch_to_lib(args.cross_arch) + if cross_arch: + deb_arch = Util.native_arch_to_deb_arch(cross_arch) + abi = Util.native_arch_to_abi(cross_arch) + lib = Util.native_arch_to_lib(cross_arch) gcc = "gcc-" + abi varmap["cross_arch"] = deb_arch pkg_names = [p + ":" + deb_arch for p in cross_pkgs.values()] @@ -749,7 +750,7 @@ class Application: sed -Ei 's,^# (en_US\\.UTF-8 .*)$,\\1,' /etc/locale.gen && \\ dpkg-reconfigure locales """).format(**varmap)) - if args.cross_arch: + if cross_arch: # Intentionally a separate RUN command from the above # so that the common packages of all cross-built images # share a Docker image layer. @@ -831,7 +832,7 @@ class Application: ENV LANG "en_US.UTF-8" """).format(**varmap)) - if args.cross_arch: + if cross_arch: sys.stdout.write(textwrap.dedent(""" ENV ABI "{cross_abi}" ENV CONFIGURE_OPTS "--host={cross_abi} \\ -- 2.24.1
Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- guests/lcitool | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/guests/lcitool b/guests/lcitool index 6624887..80a0d6a 100755 --- a/guests/lcitool +++ b/guests/lcitool @@ -725,7 +725,9 @@ class Application: varmap = {} varmap["package_manager"] = package_manager varmap["pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkgs.values()))) - varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) + + if pip_pkgs: + varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) if package_format == "deb": if cross_arch: @@ -823,7 +825,7 @@ class Application: sys.stdout.write(script.format(**varmap)) - if pip_pkgs: + if "pip_pkgs" in varmap: sys.stdout.write(textwrap.dedent(""" RUN pip3 install {pip_pkgs} """).format(**varmap)) -- 2.24.1
Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- guests/lcitool | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/guests/lcitool b/guests/lcitool index 80a0d6a..6da2206 100755 --- a/guests/lcitool +++ b/guests/lcitool @@ -720,8 +720,6 @@ class Application: pkg_align = " \\\n" + (" " * len("RUN " + package_manager + " ")) pip_pkg_align = " \\\n" + (" " * len("RUN pip3 ")) - print("FROM {}".format(facts["docker_base"])) - varmap = {} varmap["package_manager"] = package_manager varmap["pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkgs.values()))) @@ -729,19 +727,21 @@ class Application: if pip_pkgs: varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) - if package_format == "deb": - if cross_arch: - deb_arch = Util.native_arch_to_deb_arch(cross_arch) - abi = Util.native_arch_to_abi(cross_arch) - lib = Util.native_arch_to_lib(cross_arch) - gcc = "gcc-" + abi - varmap["cross_arch"] = deb_arch - pkg_names = [p + ":" + deb_arch for p in cross_pkgs.values()] - pkg_names.append(gcc) - varmap["cross_pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkg_names))) - varmap["cross_abi"] = abi - varmap["cross_lib"] = lib + if package_format == "deb" and args.cross_arch: + deb_arch = Util.native_arch_to_deb_arch(args.cross_arch) + abi = Util.native_arch_to_abi(args.cross_arch) + lib = Util.native_arch_to_lib(args.cross_arch) + gcc = "gcc-" + abi + varmap["cross_arch"] = deb_arch + pkg_names = [p + ":" + deb_arch for p in cross_pkgs.values()] + pkg_names.append(gcc) + varmap["cross_pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkg_names))) + varmap["cross_abi"] = abi + varmap["cross_lib"] = lib + print("FROM {}".format(facts["docker_base"])) + + if package_format == "deb": sys.stdout.write(textwrap.dedent(""" RUN export DEBIAN_FRONTEND=noninteractive && \\ {package_manager} update && \\ -- 2.24.1
This function takes care of approximately half of the job that was up until now handled by _action_dockerfile(), namely taking a populated dictionary and turning it into a Dockerfile. Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- guests/lcitool | 192 +++++++++++++++++++++++++------------------------ 1 file changed, 100 insertions(+), 92 deletions(-) diff --git a/guests/lcitool b/guests/lcitool index 6da2206..f216910 100755 --- a/guests/lcitool +++ b/guests/lcitool @@ -642,102 +642,11 @@ class Application: with open(keyfile, "r") as r: return r.read().rstrip() - def _action_dockerfile(self, args): - mappings = self._projects.get_mappings() - pip_mappings = self._projects.get_pip_mappings() - - hosts = self._inventory.expand_pattern(args.hosts) - if len(hosts) > 1: - raise Exception("Can't generate Dockerfile for multiple hosts") - host = hosts[0] - - facts = self._inventory.get_facts(host) + def _dockerfile_format(self, facts, cross_arch, varmap): package_format = facts["package_format"] package_manager = facts["package_manager"] os_name = facts["os_name"] os_version = facts["os_version"] - os_full = os_name + os_version - cross_arch = args.cross_arch - - if package_format not in ["deb", "rpm"]: - raise Exception("Host {} doesn't support Dockerfiles".format(host)) - if cross_arch: - if os_name != "Debian": - raise Exception("Cannot cross compile on {}".format(os_name)) - if cross_arch == self._native_arch: - raise Exception("Cross arch {} should differ from native {}". - format(cross_arch, self._native_arch)) - - projects = self._projects.expand_pattern(args.projects) - for project in projects: - if project not in facts["projects"]: - raise Exception( - "Host {} doesn't support project {}".format( - host, - project, - ) - ) - - pkgs = {} - cross_pkgs = {} - pip_pkgs = {} - base_keys = ["default", package_format, os_name, os_full] - cross_keys = [] - if cross_arch: - keys = base_keys + [cross_arch + "-" + k for k in base_keys] - cross_keys = ["cross-policy-" + k for k in base_keys] - else: - keys = base_keys + [self._native_arch + "-" + k for k in base_keys] - - # We need to add the base project manually here: the standard - # machinery hides it because it's an implementation detail - for project in projects + ["base"]: - for package in self._projects.get_packages(project): - cross_policy = "native" - for key in cross_keys: - if key in mappings[package]: - cross_policy = mappings[package][key] - if cross_policy not in ["native", "foreign", "skip"]: - raise Exception( - "Unexpected cross arch policy {} for {}".format - (cross_policy, package)) - - for key in keys: - if key in mappings[package]: - pkgs[package] = mappings[package][key] - if package in pip_mappings and key in pip_mappings[package]: - pip_pkgs[package] = pip_mappings[package][key] - - if package not in pkgs: - continue - if package in pip_pkgs and pkgs[package] is not None: - del pip_pkgs[package] - if cross_policy == "foreign" and pkgs[package] is not None: - cross_pkgs[package] = pkgs[package] - if pkgs[package] is None or cross_policy in ["skip", "foreign"]: - del pkgs[package] - - pkg_align = " \\\n" + (" " * len("RUN " + package_manager + " ")) - pip_pkg_align = " \\\n" + (" " * len("RUN pip3 ")) - - varmap = {} - varmap["package_manager"] = package_manager - varmap["pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkgs.values()))) - - if pip_pkgs: - varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) - - if package_format == "deb" and args.cross_arch: - deb_arch = Util.native_arch_to_deb_arch(args.cross_arch) - abi = Util.native_arch_to_abi(args.cross_arch) - lib = Util.native_arch_to_lib(args.cross_arch) - gcc = "gcc-" + abi - varmap["cross_arch"] = deb_arch - pkg_names = [p + ":" + deb_arch for p in cross_pkgs.values()] - pkg_names.append(gcc) - varmap["cross_pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkg_names))) - varmap["cross_abi"] = abi - varmap["cross_lib"] = lib print("FROM {}".format(facts["docker_base"])) @@ -842,6 +751,105 @@ class Application: ENV PKG_CONFIG_LIBDIR "/usr/lib/{cross_lib}/pkgconfig" """).format(**varmap)) + def _action_dockerfile(self, args): + mappings = self._projects.get_mappings() + pip_mappings = self._projects.get_pip_mappings() + + hosts = self._inventory.expand_pattern(args.hosts) + if len(hosts) > 1: + raise Exception("Can't generate Dockerfile for multiple hosts") + host = hosts[0] + + facts = self._inventory.get_facts(host) + package_format = facts["package_format"] + package_manager = facts["package_manager"] + os_name = facts["os_name"] + os_version = facts["os_version"] + os_full = os_name + os_version + cross_arch = args.cross_arch + + if package_format not in ["deb", "rpm"]: + raise Exception("Host {} doesn't support Dockerfiles".format(host)) + if cross_arch: + if os_name != "Debian": + raise Exception("Cannot cross compile on {}".format(os_name)) + if cross_arch == self._native_arch: + raise Exception("Cross arch {} should differ from native {}". + format(cross_arch, self._native_arch)) + + projects = self._projects.expand_pattern(args.projects) + for project in projects: + if project not in facts["projects"]: + raise Exception( + "Host {} doesn't support project {}".format( + host, + project, + ) + ) + + pkgs = {} + cross_pkgs = {} + pip_pkgs = {} + base_keys = ["default", package_format, os_name, os_full] + cross_keys = [] + if cross_arch: + keys = base_keys + [cross_arch + "-" + k for k in base_keys] + cross_keys = ["cross-policy-" + k for k in base_keys] + else: + keys = base_keys + [self._native_arch + "-" + k for k in base_keys] + + # We need to add the base project manually here: the standard + # machinery hides it because it's an implementation detail + for project in projects + ["base"]: + for package in self._projects.get_packages(project): + cross_policy = "native" + for key in cross_keys: + if key in mappings[package]: + cross_policy = mappings[package][key] + if cross_policy not in ["native", "foreign", "skip"]: + raise Exception( + "Unexpected cross arch policy {} for {}".format + (cross_policy, package)) + + for key in keys: + if key in mappings[package]: + pkgs[package] = mappings[package][key] + if package in pip_mappings and key in pip_mappings[package]: + pip_pkgs[package] = pip_mappings[package][key] + + if package not in pkgs: + continue + if package in pip_pkgs and pkgs[package] is not None: + del pip_pkgs[package] + if cross_policy == "foreign" and pkgs[package] is not None: + cross_pkgs[package] = pkgs[package] + if pkgs[package] is None or cross_policy in ["skip", "foreign"]: + del pkgs[package] + + pkg_align = " \\\n" + (" " * len("RUN " + package_manager + " ")) + pip_pkg_align = " \\\n" + (" " * len("RUN pip3 ")) + + varmap = {} + varmap["package_manager"] = package_manager + varmap["pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkgs.values()))) + + if package_format == "deb" and cross_arch: + deb_arch = Util.native_arch_to_deb_arch(cross_arch) + abi = Util.native_arch_to_abi(cross_arch) + lib = Util.native_arch_to_lib(cross_arch) + gcc = "gcc-" + abi + varmap["cross_arch"] = deb_arch + pkg_names = [p + ":" + deb_arch for p in cross_pkgs.values()] + pkg_names.append(gcc) + varmap["cross_pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkg_names))) + varmap["cross_abi"] = abi + varmap["cross_lib"] = lib + + if pip_pkgs: + varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) + + self._dockerfile_format(facts, cross_arch, varmap) + def run(self): args = self._parser.parse_args() if args.debug: -- 2.24.1
This function takes care of most of what's still handled by _action_dockerfile(), namely populating the dictionary that _dockerfile_format() will later consume. Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- guests/lcitool | 132 ++++++++++++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 61 deletions(-) diff --git a/guests/lcitool b/guests/lcitool index f216910..82e98c5 100755 --- a/guests/lcitool +++ b/guests/lcitool @@ -642,6 +642,76 @@ class Application: with open(keyfile, "r") as r: return r.read().rstrip() + def _dockerfile_build_varmap(self, facts, mappings, pip_mappings, projects, cross_arch): + package_format = facts["package_format"] + package_manager = facts["package_manager"] + os_name = facts["os_name"] + os_version = facts["os_version"] + os_full = os_name + os_version + + pkgs = {} + cross_pkgs = {} + pip_pkgs = {} + base_keys = ["default", package_format, os_name, os_full] + cross_keys = [] + if cross_arch: + keys = base_keys + [cross_arch + "-" + k for k in base_keys] + cross_keys = ["cross-policy-" + k for k in base_keys] + else: + keys = base_keys + [self._native_arch + "-" + k for k in base_keys] + + # We need to add the base project manually here: the standard + # machinery hides it because it's an implementation detail + for project in projects + ["base"]: + for package in self._projects.get_packages(project): + cross_policy = "native" + for key in cross_keys: + if key in mappings[package]: + cross_policy = mappings[package][key] + if cross_policy not in ["native", "foreign", "skip"]: + raise Exception( + "Unexpected cross arch policy {} for {}".format + (cross_policy, package)) + + for key in keys: + if key in mappings[package]: + pkgs[package] = mappings[package][key] + if package in pip_mappings and key in pip_mappings[package]: + pip_pkgs[package] = pip_mappings[package][key] + + if package not in pkgs: + continue + if package in pip_pkgs and pkgs[package] is not None: + del pip_pkgs[package] + if cross_policy == "foreign" and pkgs[package] is not None: + cross_pkgs[package] = pkgs[package] + if pkgs[package] is None or cross_policy in ["skip", "foreign"]: + del pkgs[package] + + pkg_align = " \\\n" + (" " * len("RUN " + package_manager + " ")) + pip_pkg_align = " \\\n" + (" " * len("RUN pip3 ")) + + varmap = {} + varmap["package_manager"] = package_manager + varmap["pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkgs.values()))) + + if package_format == "deb" and cross_arch: + deb_arch = Util.native_arch_to_deb_arch(cross_arch) + abi = Util.native_arch_to_abi(cross_arch) + lib = Util.native_arch_to_lib(cross_arch) + gcc = "gcc-" + abi + varmap["cross_arch"] = deb_arch + pkg_names = [p + ":" + deb_arch for p in cross_pkgs.values()] + pkg_names.append(gcc) + varmap["cross_pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkg_names))) + varmap["cross_abi"] = abi + varmap["cross_lib"] = lib + + if pip_pkgs: + varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) + + return varmap + def _dockerfile_format(self, facts, cross_arch, varmap): package_format = facts["package_format"] package_manager = facts["package_manager"] @@ -787,67 +857,7 @@ class Application: ) ) - pkgs = {} - cross_pkgs = {} - pip_pkgs = {} - base_keys = ["default", package_format, os_name, os_full] - cross_keys = [] - if cross_arch: - keys = base_keys + [cross_arch + "-" + k for k in base_keys] - cross_keys = ["cross-policy-" + k for k in base_keys] - else: - keys = base_keys + [self._native_arch + "-" + k for k in base_keys] - - # We need to add the base project manually here: the standard - # machinery hides it because it's an implementation detail - for project in projects + ["base"]: - for package in self._projects.get_packages(project): - cross_policy = "native" - for key in cross_keys: - if key in mappings[package]: - cross_policy = mappings[package][key] - if cross_policy not in ["native", "foreign", "skip"]: - raise Exception( - "Unexpected cross arch policy {} for {}".format - (cross_policy, package)) - - for key in keys: - if key in mappings[package]: - pkgs[package] = mappings[package][key] - if package in pip_mappings and key in pip_mappings[package]: - pip_pkgs[package] = pip_mappings[package][key] - - if package not in pkgs: - continue - if package in pip_pkgs and pkgs[package] is not None: - del pip_pkgs[package] - if cross_policy == "foreign" and pkgs[package] is not None: - cross_pkgs[package] = pkgs[package] - if pkgs[package] is None or cross_policy in ["skip", "foreign"]: - del pkgs[package] - - pkg_align = " \\\n" + (" " * len("RUN " + package_manager + " ")) - pip_pkg_align = " \\\n" + (" " * len("RUN pip3 ")) - - varmap = {} - varmap["package_manager"] = package_manager - varmap["pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkgs.values()))) - - if package_format == "deb" and cross_arch: - deb_arch = Util.native_arch_to_deb_arch(cross_arch) - abi = Util.native_arch_to_abi(cross_arch) - lib = Util.native_arch_to_lib(cross_arch) - gcc = "gcc-" + abi - varmap["cross_arch"] = deb_arch - pkg_names = [p + ":" + deb_arch for p in cross_pkgs.values()] - pkg_names.append(gcc) - varmap["cross_pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkg_names))) - varmap["cross_abi"] = abi - varmap["cross_lib"] = lib - - if pip_pkgs: - varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) - + varmap = self._dockerfile_build_varmap(facts, mappings, pip_mappings, projects, cross_arch) self._dockerfile_format(facts, cross_arch, varmap) def run(self): -- 2.24.1
A lot of the code in _dockerfile_build_varmap() only applies to Debian, so we can have a simpler version for use with Fedora. This split will come in handy later on, when we start introducing features that work in Fedora and not in Debian. Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- guests/lcitool | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/guests/lcitool b/guests/lcitool index 82e98c5..37a0253 100755 --- a/guests/lcitool +++ b/guests/lcitool @@ -643,6 +643,12 @@ class Application: return r.read().rstrip() def _dockerfile_build_varmap(self, facts, mappings, pip_mappings, projects, cross_arch): + if facts["package_format"] == "deb": + return self._dockerfile_build_varmap_deb(facts, mappings, pip_mappings, projects, cross_arch) + if facts["package_format"] == "rpm": + return self._dockerfile_build_varmap_rpm(facts, mappings, pip_mappings, projects, cross_arch) + + def _dockerfile_build_varmap_deb(self, facts, mappings, pip_mappings, projects, cross_arch): package_format = facts["package_format"] package_manager = facts["package_manager"] os_name = facts["os_name"] @@ -695,7 +701,7 @@ class Application: varmap["package_manager"] = package_manager varmap["pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkgs.values()))) - if package_format == "deb" and cross_arch: + if cross_arch: deb_arch = Util.native_arch_to_deb_arch(cross_arch) abi = Util.native_arch_to_abi(cross_arch) lib = Util.native_arch_to_lib(cross_arch) @@ -712,6 +718,46 @@ class Application: return varmap + def _dockerfile_build_varmap_rpm(self, facts, mappings, pip_mappings, projects, cross_arch): + package_format = facts["package_format"] + package_manager = facts["package_manager"] + os_name = facts["os_name"] + os_version = facts["os_version"] + os_full = os_name + os_version + + pkgs = {} + pip_pkgs = {} + keys = ["default", package_format, os_name, os_full] + + # We need to add the base project manually here: the standard + # machinery hides it because it's an implementation detail + for project in projects + ["base"]: + for package in self._projects.get_packages(project): + for key in keys: + if key in mappings[package]: + pkgs[package] = mappings[package][key] + if package in pip_mappings and key in pip_mappings[package]: + pip_pkgs[package] = pip_mappings[package][key] + + if package not in pkgs: + continue + if package in pip_pkgs and pkgs[package] is not None: + del pip_pkgs[package] + if pkgs[package] is None: + del pkgs[package] + + pkg_align = " \\\n" + (" " * len("RUN " + package_manager + " ")) + pip_pkg_align = " \\\n" + (" " * len("RUN pip3 ")) + + varmap = {} + varmap["package_manager"] = package_manager + varmap["pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkgs.values()))) + + if pip_pkgs: + varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) + + return varmap + def _dockerfile_format(self, facts, cross_arch, varmap): package_format = facts["package_format"] package_manager = facts["package_manager"] -- 2.24.1
Up until this point, MinGW has been treated as a very special case, even having its own sub-projects. While this was perfectly reasonable back when lcitool was introduced, now that we support cross-building as a first-class concept in the Dockerfile generator it makes little sense for MinGW builds not to make use of it. Note that, as of this commit, MinGW is still considered somewhat special: in particular, only the Dockerfile generator can treat it like any other architecture, and everything else still needs to use the sub-projects. In the longer run we'll want to expand the first-class cross-building support to the rest of lcitool; this is merely another step in that direction. Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- guests/lcitool | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/guests/lcitool b/guests/lcitool index 37a0253..abc803b 100755 --- a/guests/lcitool +++ b/guests/lcitool @@ -93,6 +93,8 @@ class Util: "armv6l": "arm-linux-gnueabi", "armv7l": "arm-linux-gnueabihf", "i686": "i686-linux-gnu", + "mingw32": "i686-w64-mingw32", + "mingw64": "x86_64-w64-mingw32", "mips": "mips-linux-gnu", "mipsel": "mipsel-linux-gnu", "mips64el": "mips64el-linux-gnuabi64", @@ -726,6 +728,7 @@ class Application: os_full = os_name + os_version pkgs = {} + cross_pkgs = {} pip_pkgs = {} keys = ["default", package_format, os_name, os_full] @@ -746,6 +749,24 @@ class Application: if pkgs[package] is None: del pkgs[package] + if cross_arch: + cross_projects = [] + for project in projects: + cross_project = project + "+" + cross_arch + if cross_project in facts["projects"]: + cross_projects.extend([cross_project]) + + for project in cross_projects: + for package in self._projects.get_packages(project): + for key in keys: + if key in mappings[package]: + cross_pkgs[package] = mappings[package][key] + + if package not in cross_pkgs: + continue + if cross_pkgs[package] is None: + del cross_pkgs[package] + pkg_align = " \\\n" + (" " * len("RUN " + package_manager + " ")) pip_pkg_align = " \\\n" + (" " * len("RUN pip3 ")) @@ -753,6 +774,10 @@ class Application: varmap["package_manager"] = package_manager varmap["pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(pkgs.values()))) + if cross_arch: + varmap["cross_abi"] = Util.native_arch_to_abi(cross_arch) + varmap["cross_pkgs"] = pkg_align[1:] + pkg_align.join(sorted(set(cross_pkgs.values()))) + if pip_pkgs: varmap["pip_pkgs"] = pip_pkg_align[1:] + pip_pkg_align.join(sorted(set(pip_pkgs.values()))) @@ -850,6 +875,15 @@ class Application: sys.stdout.write(script.format(**varmap)) + if cross_arch: + # Intentionally a separate RUN command from the above + # so that the common packages of all cross-built images + # share a Docker image layer. + sys.stdout.write(textwrap.dedent(""" + RUN {package_manager} install -y {cross_pkgs} && \\ + {package_manager} clean all -y + """).format(**varmap)) + if "pip_pkgs" in varmap: sys.stdout.write(textwrap.dedent(""" RUN pip3 install {pip_pkgs} @@ -859,7 +893,7 @@ class Application: ENV LANG "en_US.UTF-8" """).format(**varmap)) - if cross_arch: + if cross_arch and package_format == "deb": sys.stdout.write(textwrap.dedent(""" ENV ABI "{cross_abi}" ENV CONFIGURE_OPTS "--host={cross_abi} \\ @@ -867,6 +901,14 @@ class Application: ENV PKG_CONFIG_LIBDIR "/usr/lib/{cross_lib}/pkgconfig" """).format(**varmap)) + if cross_arch and package_format == "rpm": + sys.stdout.write(textwrap.dedent(""" + ENV ABI "{cross_abi}" + ENV CONFIGURE_OPTS "--host={cross_abi} \\ + --target={cross_abi}" + ENV PKG_CONFIG_LIBDIR "/usr/{cross_abi}/sys-root/mingw/lib/pkgconfig:/usr/{cross_abi}/sys-root/mingw/share/pkgconfig" + """).format(**varmap)) + def _action_dockerfile(self, args): mappings = self._projects.get_mappings() pip_mappings = self._projects.get_pip_mappings() @@ -887,7 +929,7 @@ class Application: if package_format not in ["deb", "rpm"]: raise Exception("Host {} doesn't support Dockerfiles".format(host)) if cross_arch: - if os_name != "Debian": + if os_name not in ["Debian", "Fedora"]: raise Exception("Cannot cross compile on {}".format(os_name)) if cross_arch == self._native_arch: raise Exception("Cross arch {} should differ from native {}". -- 2.24.1
There are a few problematic scenarios that we can catch early. Signed-off-by: Andrea Bolognani <abologna@redhat.com> --- guests/lcitool | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/guests/lcitool b/guests/lcitool index abc803b..771402e 100755 --- a/guests/lcitool +++ b/guests/lcitool @@ -931,12 +931,28 @@ class Application: if cross_arch: if os_name not in ["Debian", "Fedora"]: raise Exception("Cannot cross compile on {}".format(os_name)) + if os_name == "Debian" and cross_arch.startswith("mingw"): + raise Exception( + "Cannot cross compile for {} on {}".format( + cross_arch, + os_name, + ) + ) + if os_name == "Fedora" and not cross_arch.startswith("mingw"): + raise Exception( + "Cannot cross compile for {} on {}".format( + cross_arch, + os_name, + ) + ) if cross_arch == self._native_arch: raise Exception("Cross arch {} should differ from native {}". format(cross_arch, self._native_arch)) projects = self._projects.expand_pattern(args.projects) for project in projects: + if project.rfind("+mingw") >= 0: + raise Exception("Obsolete syntax, please use --cross-arch") if project not in facts["projects"]: raise Exception( "Host {} doesn't support project {}".format( -- 2.24.1
On Mon, 2020-02-10 at 18:18 +0100, Andrea Bolognani wrote:
More details in the commit message for patch 7/8.
Pavel pointed out today that the current method of triggering MinGW builds using our CI scaffolding, eg.
$ make ci-build@fedora-30 CI_CONFIGURE=mingw64-configure
is easy to get wrong and not very discoverable, so I took that as motivation to implement a change that I had been thinking about for a long time anyway. The new usage will be
$ make ci-build@fedora-30-cross-mingw64
which aligns with how we're already doing cross-builds for other architectures and is discoverable via 'make ci-list-images'.
The implementation is not the prettiest, but the Dockerfile generator in general could use some love so I don't think this improvement should be blocked because of that; I'll try to spend some time refactoring and cleaning up once this has been merged.
Andrea Bolognani (8): lcitool: Introduce cross_arch local variable lcitool: Change check for pip_pkgs formatting lcitool: Separate computation and formatting lcitool: Introduce _dockerfile_format() lcitool: Introduce _dockerfile_build_varmap() lcitool: Add RPM-specific _dockerfile_build_varmap() variant lcitool: Support MinGW cross-build Dockerfiles on Fedora lcitool: Add more checks to _action_dockerfile()
guests/lcitool | 219 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 172 insertions(+), 47 deletions(-)
Ping? :) -- Andrea Bolognani / Red Hat / Virtualization
On Mon, Feb 10, 2020 at 06:18:09PM +0100, Andrea Bolognani wrote:
More details in the commit message for patch 7/8.
Pavel pointed out today that the current method of triggering MinGW builds using our CI scaffolding, eg.
$ make ci-build@fedora-30 CI_CONFIGURE=mingw64-configure
is easy to get wrong and not very discoverable, so I took that as motivation to implement a change that I had been thinking about for a long time anyway. The new usage will be
$ make ci-build@fedora-30-cross-mingw64
which aligns with how we're already doing cross-builds for other architectures and is discoverable via 'make ci-list-images'.
The implementation is not the prettiest, but the Dockerfile generator in general could use some love so I don't think this improvement should be blocked because of that; I'll try to spend some time refactoring and cleaning up once this has been merged.
Andrea Bolognani (8): lcitool: Introduce cross_arch local variable lcitool: Change check for pip_pkgs formatting lcitool: Separate computation and formatting lcitool: Introduce _dockerfile_format() lcitool: Introduce _dockerfile_build_varmap() lcitool: Add RPM-specific _dockerfile_build_varmap() variant lcitool: Support MinGW cross-build Dockerfiles on Fedora lcitool: Add more checks to _action_dockerfile()
guests/lcitool | 219 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 172 insertions(+), 47 deletions(-)
Reviewed-by: Ján Tomko <jtomko@redhat.com> Jano
participants (2)
-
Andrea Bolognani -
Ján Tomko