Parts of the code can be reused by another registry checker script introduced in the next patch. Signed-off-by: Erik Skultety <eskultet(a)redhat.com&gt; --- ci/containers/util.py | 25 +++++++++++++++++++++++++ ci/list-images.py | 43 +++++++++++++++++++++++-------------------- 2 files changed, 48 insertions(+), 20 deletions(-) create mode 100644 ci/containers/util.py diff --git a/ci/containers/util.py b/ci/containers/util.py new file mode 100644 index 0000000000..1ce326bba2 --- /dev/null +++ b/ci/containers/util.py @@ -0,0 +1,25 @@ +import json +import urllib.request as urllib + + +PROJECT_ID = 192693 + + +def get_registry_uri(project_id, + base_uri="https://gitlab.com", + api_version=4): + uri = f"{base_uri}/api/v{str(api_version)}/projects/{project_id}/registry/repositories" + return uri + + +def list_images(uri): + """ + Returns all container images as currently available for the given GitLab + project. + + ret: list of container image names + """ + r = urllib.urlopen(uri) + + # read the HTTP response and load the JSON part of it + return json.loads(r.read().decode()) diff --git a/ci/list-images.py b/ci/list-images.py index 6862ac347f..1a1d87c10b 100644 --- a/ci/list-images.py +++ b/ci/list-images.py @@ -1,29 +1,32 @@ #!/usr/bin/env python3 -import json -import urllib.request as urllib +import containers.util as util -PROJECT_ID = 192693 -DOMAIN = "https://gitlab.com" -API = "api/v4" -uri = f"{DOMAIN}/{API}/projects/{PROJECT_ID}/registry/repositories" -r = urllib.urlopen(uri + "?per_page=100") +def list_image_names(uri): + images = util.list_images(uri) -# read the HTTP response and load the JSON part of it -data = json.loads(r.read().decode()) + # skip the "ci-" prefix each of our container images' name has + names = [i["name"][3:] for i in images] + names.sort() -# skip the "ci-" prefix each of our container images' name has -names = [i["name"][3:] for i in data] -names.sort() + return names -names_native = [name for name in names if "cross" not in name] -names_cross = [name for name in names if "cross" in name] -print("Available x86 container images:\n") -print("\t" + "\n\t".join(names_native)) +def main(): + names = list_image_names(util.get_registry_uri(util.PROJECT_ID) + "?per_page=100") -if names_cross: - print() - print("Available cross-compiler container images:\n") - print("\t" + "\n\t".join(names_cross)) + names_native = [name for name in names if "cross" not in name] + names_cross = [name for name in names if "cross" in name] + + print("Available x86 container images:\n") + print("\t" + "\n\t".join(names_native)) + + if names_cross: + print() + print("Available cross-compiler container images:\n") + print("\t" + "\n\t".join(names_cross)) + + +if __name__ == "__main__": + main() -- 2.29.2

When removing the dependency on sudo in our CI Makefile, I realized that the list of images that we pull contains some old images that we no longer support in lcitool, but can be still pulled from the registry. This is a simple checker which runs in context of the refresh script which informs the developer/maintainer that there are some old redundant images that should be purged from the registry, so that people don't try to run local builds in those (which will most likely fail anyway). Signed-off-by: Erik Skultety <eskultet(a)redhat.com&gt; --- ci/containers/check-registry.py | 96 +++++++++++++++++++++++++++++++++ ci/containers/refresh | 5 ++ 2 files changed, 101 insertions(+) create mode 100644 ci/containers/check-registry.py diff --git a/ci/containers/check-registry.py b/ci/containers/check-registry.py new file mode 100644 index 0000000000..f0159da54d --- /dev/null +++ b/ci/containers/check-registry.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import argparse +import subprocess +import shutil +import sys + +import util + + +def get_image_distro(image_name: str): + name_prefix = "ci-" + name_suffix = "-cross-" + + distro = image_name[len(name_prefix):] + + index = distro.find(name_suffix) + if index > 0: + distro = distro[:index] + + return distro + + +def get_undesirables(registry_distros_d, hosts_l): + """ Returns a dictionary of 'id':'name' pairs of images that can be purged""" + + undesirables_d = {} + for distro in registry_distros_d: + if distro not in hosts_l: + for image in registry_distros_d[distro]: + undesirables_d[str(image["id"])] = image["path"] + + return undesirables_d if undesirables_d else None + + +def get_lcitool_hosts(lcitool_path): + """ Returns a list of supported hosts by lcitool + @param lcitool_path: absolute path to local copy of lcitool, if omitted it is + assumed lcitool is installed in the current environment""" + + if not lcitool_path: + lcitool_path = "lcitool" + if not shutil.which(lcitool_path): + sys.exit("error: 'lcitool' not installed") + + lcitool_out = subprocess.check_output([lcitool_path, "hosts"]) + return lcitool_out.decode("utf-8").splitlines() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("project_id", + help="GitLab project ID") + parser.add_argument("--lcitool_path", + dest="lcitool", + metavar="PATH", + help="absolute path to lcitool, CWD is used if omitted") + + args = parser.parse_args() + + uri = util.get_registry_uri(util.PROJECT_ID) + images_json = util.list_images(uri + "?per_page=100") + + registry_distros_d = {} + for image in images_json: + distro_name = get_image_distro(image["name"]) + + try: + registry_distros_d[distro_name].append(image) + except KeyError: + registry_distros_d[distro_name] = [image] + + hosts_l = get_lcitool_hosts(args.lcitool) + + # print the list of images that we can safely purge from the registry + undesirables_d = get_undesirables(registry_distros_d, hosts_l) + if undesirables_d: + undesirable_image_names = "\t" + "\n\t".join(undesirables_d.values()) + undesirable_image_ids = " ".join(undesirables_d.keys()) + + sys.exit(f""" +The following images can be purged from the registry: + +{undesirable_image_names} + +You can remove the above images over the API with the following code snippet: + +\t$ for image_id in {undesirable_image_ids} \\ +\t;do \\ +\t\tcurl --request DELETE --header "PRIVATE-TOKEN: <access_token>" \\ +\t\t{util.get_registry_uri(util.PROJECT_ID)}/$image_id \\ +\t;done""") + + +if __name__ == "__main__": + main() diff --git a/ci/containers/refresh b/ci/containers/refresh index a39d0b0996..9dfaf75300 100755 --- a/ci/containers/refresh +++ b/ci/containers/refresh @@ -40,3 +40,8 @@ do $LCITOOL dockerfile $host libvirt > ci-$host.Dockerfile done + +# Check whether there are some old images in the container registry that could +# be purged - mainly because the list-images script would list them as +# available +/usr/bin/env python3 check-registry.py --lcitool_path "$LCITOOL" 192693 -- 2.29.2