[libvirt] [PATCH sandbox 00/24] Add support for docker v2 registry

Daniel P. Berrange

15 Jul 2016 15 Jul '16

9:07 a.m.

I was on the verge of doing a release of libvirt-sandbox to include the docker/virt-builder support when I discovered it didn't work with many new images since docker is only exposing them over its v2 registry API. Hence this series to add support for the v2 API. It also includes a tonne of fixes for bugs encountered during debugging, and adds a new 'list' command for virt-sandbox-image so you can actally see what you have downloaded :-) Daniel P. Berrange (24): Remove transfer annotations on scalar parameters Improve error message for unsupported URIs init: search $PATH for user specified binary image: rename python source impl modules image: remove execute permission from cli.py image: add support for --debug arg image: don't assume ValueError has a multi-line message image: check for errors running mkfs / virt-sandbox virt-builder: check exit status of commands run virt-builder: disable setup of network for virt-builder docker: check exit status of qemu-img command docker: rename destdir to templatedir docker: Remove duplicated call to download_template docker: add missing hashlib import statement docker: fix download progress target value docker: remove unused variables & code in download method docker: introduce a DockerImage class docker: fix setting of Accept header docker: refactor authentication method handling docker: introduce a DockerRegistry class docker: refactor download method exception handling docker: implement support for oauth docker: add support for v2 registry server image: add 'list' command for viewing local templates libvirt-sandbox/image/cli.py | 76 ++- libvirt-sandbox/image/sources/DockerSource.py | 407 ------------ libvirt-sandbox/image/sources/Makefile.am | 6 +- .../image/sources/{Source.py => base.py} | 14 +- libvirt-sandbox/image/sources/docker.py | 683 +++++++++++++++++++++ .../{VirtBuilderSource.py => virtbuilder.py} | 34 +- libvirt-sandbox/image/template.py | 32 +- .../libvirt-sandbox-config-interactive.c | 4 +- .../libvirt-sandbox-config-network-route.c | 2 +- libvirt-sandbox/libvirt-sandbox-config.c | 16 +- .../libvirt-sandbox-context-interactive.c | 6 +- libvirt-sandbox/libvirt-sandbox-init-common.c | 2 +- 12 files changed, 818 insertions(+), 464 deletions(-) mode change 100755 => 100644 libvirt-sandbox/image/cli.py delete mode 100644 libvirt-sandbox/image/sources/DockerSource.py rename libvirt-sandbox/image/sources/{Source.py => base.py} (93%) create mode 100644 libvirt-sandbox/image/sources/docker.py rename libvirt-sandbox/image/sources/{VirtBuilderSource.py => virtbuilder.py} (76%) -- 2.7.4

Show replies by date

Daniel P. Berrange

Cedric Bosdonnat

18 Jul 18 Jul

5:26 a.m.

New subject: [libvirt] [PATCH sandbox 24/24] image: add 'list' command for viewing local templates

On Fri, 2016-07-15 at 14:08 +0100, Daniel P. Berrange wrote:

...

Introduce a command able to list locally stored image templates:

$ virt-sandbox-image list docker:library/ubuntu?tag=14.04.1 docker:library/ubuntu?tag=14.04.2 virt-builder:/fedora-23

or restrict to a single source type

$ virt-sandbox-image list --source docker docker:library/ubuntu?tag=14.04.1 docker:library/ubuntu?tag=14.04.2

Signed-off-by: Daniel P. Berrange <berrange@redhat.com> --- libvirt-sandbox/image/cli.py | 24 ++++++++++++++++++++++ libvirt-sandbox/image/sources/base.py | 10 ++++++++++ libvirt-sandbox/image/sources/docker.py | 22 ++++++++++++++++++++ libvirt-sandbox/image/sources/virtbuilder.py | 18 +++++++++++++++++ libvirt-sandbox/image/template.py | 30 +++++++++++++++++++--------- 5 files changed, 95 insertions(+), 9 deletions(-)

diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py index b0d864f..66854e4 100644 --- a/libvirt-sandbox/image/cli.py +++ b/libvirt-sandbox/image/cli.py @@ -135,6 +135,18 @@ def run(args): os.unlink(diskfile) source.post_run(tmpl, template_dir, name)

+def list_cached(args): + tmpls = [] + if args.source is not None: + tmpls.extend(template.Template.get_all(args.source, + "%s/%s" % (args.template_dir, args.source))) + else: + for source in ["docker", "virt-builder"]: + tmpls.extend(template.Template.get_all(source, + "%s/%s" % (args.template_dir, source))) + for tmpl in tmpls: + print tmpl + def requires_template(parser): parser.add_argument("template", help=_("URI of the template")) @@ -221,6 +233,17 @@ def gen_run_args(subparser):

parser.set_defaults(func=run)

+def gen_list_args(subparser): + parser = gen_command_parser(subparser, "list", + _("List locally cached images")) + requires_debug(parser) + requires_template_dir(parser) + + parser.add_argument("-s","--source", + help=_("Name of the template source")) + + parser.set_defaults(func=list_cached) + def main(): parser = argparse.ArgumentParser(description="Sandbox Container Image Tool")

@@ -228,6 +251,7 @@ def main(): gen_delete_args(subparser) gen_create_args(subparser) gen_run_args(subparser) + gen_list_args(subparser)

args = parser.parse_args() if args.debug: diff --git a/libvirt-sandbox/image/sources/base.py b/libvirt-sandbox/image/sources/base.py index f70551d..e4e4e41 100644 --- a/libvirt-sandbox/image/sources/base.py +++ b/libvirt-sandbox/image/sources/base.py @@ -34,6 +34,16 @@ class Source(): def __init__(self): pass

+ @abstractmethod + def list_templates(self, templatedir): + """ + :param templatedir: local directory path in which to store the template + + Get a list of all templates that are locally cached + + :returns: a list of libvirt_sandbox.template.Template objects + """ + pass

@abstractmethod def has_template(self, template, templatedir): diff --git a/libvirt-sandbox/image/sources/docker.py b/libvirt-sandbox/image/sources/docker.py index 291a305..dd72db7 100644 --- a/libvirt-sandbox/image/sources/docker.py +++ b/libvirt-sandbox/image/sources/docker.py @@ -32,6 +32,7 @@ import urlparse import hashlib from abc import ABCMeta, abstractmethod import copy +from libvirt_sandbox.image.template import Template

from . import base

@@ -360,6 +361,27 @@ class DockerSource(base.Source): except Exception: return False

+ def list_templates(self, templatedir): + indexes = [] + imagedirs = os.listdir(templatedir) + for imagetagid in imagedirs: + indexfile = templatedir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile,"r") as f: + index = json.load(f) + indexes.append(index) + + return [ + Template(source="docker", + protocol=None, + hostname=None, + port=None, + username=None, + password=None, + path="/%s/%s" % (index.get("repo", "library"), index["name"]), + params={ + "tag": index.get("tag", "latest"), + }) for index in indexes]

def has_template(self, template, templatedir): try: diff --git a/libvirt-sandbox/image/sources/virtbuilder.py b/libvirt-sandbox/image/sources/virtbuilder.py index 6e71b36..fefe0dd 100644 --- a/libvirt-sandbox/image/sources/virtbuilder.py +++ b/libvirt-sandbox/image/sources/virtbuilder.py @@ -24,6 +24,7 @@ import os.path import subprocess

from . import base +from libvirt_sandbox.image.template import Template

class VirtBuilderSource(base.Source): @@ -65,6 +66,23 @@ class VirtBuilderSource(base.Source): os.unlink(imagepath_original) os.unlink(tarfile)

+ def list_templates(self, templatedir): + files = [] + imagefiles = os.listdir(templatedir) + for filename in imagefiles: + if not filename.endswith(".qcow2"): + continue + files.append(filename[0:-6]) + + return [ + Template(source="virt-builder", + protocol=None, + hostname=None, + port=None, + username=None, + password=None, + path="/%s" % filename, + params={}) for filename in files]

def delete_template(self, template, templatedir): os.unlink("%s/%s.qcow2" % (templatedir, self._get_template_name(template))) diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py index 751cd4b..79dc33d 100644 --- a/libvirt-sandbox/image/template.py +++ b/libvirt-sandbox/image/template.py @@ -58,22 +58,27 @@ class Template(object): if self.params is None: self.params = {}

- def get_source_impl(self): - if self.source == "": - raise Exception("Missing scheme in image URI") - + @classmethod + def _get_source_impl(klass, source): try: p = re.compile("\W") - sourcemod = "".join(p.split(self.source)) - sourcename = "".join([i.capitalize() for i in p.split(self.source)]) + sourcemod = "".join(p.split(source)) + sourcename = "".join([i.capitalize() for i in p.split(source)])

mod = importlib.import_module( "libvirt_sandbox.image.sources." + sourcemod) classname = sourcename + "Source" classimpl = getattr(mod, classname) return classimpl() - except Exception: - raise Exception("Invalid source: '%s'" % self.source) + except Exception as e: + print e + raise Exception("Invalid source: '%s'" % source) + + def get_source_impl(self): + if self.source == "": + raise Exception("Missing scheme in image URI") + + return self._get_source_impl(self.source)

def __repr__(self): if self.protocol is not None: @@ -96,7 +101,8 @@ class Template(object): netloc = None

query = "&".join([key + "=" + self.params[key] for key in self.params.keys()]) - return urlparse.urlunparse((scheme, netloc, self.path, None, query, None)) + ret = urlparse.urlunparse((scheme, netloc, self.path, None, query, None)) + return ret

@classmethod def from_uri(klass, uri): @@ -119,3 +125,9 @@ class Template(object): o.hostname, o.port, o.username, o.password, o.path, query) + + @classmethod + def get_all(klass, source, templatedir): + impl = klass._get_source_impl(source) + + return impl.list_templates(templatedir)

The whole series looks good to me. ACK. -- Cedric

3338

Age (days ago)

3341

Last active (days ago)

List overview

Download

25 comments

2 participants

participants (2)

Cedric Bosdonnat
Daniel P. Berrange