[libvirt] [PATCH sandbox] virt-sandbox-image: switch to use URI to identify templates

Currently the CLI syntax is somewhat docker specific requiring inclusion of --registry arg to identify the docker download server. Other app containers have a notion of download server, but don't separate it from the template name. This patch removes that docker-ism by changing to use a URI for identifying the template image. So instead of virt-sandbox-image download \ --source docker --registry index.docker.io --username dan --password 123456 ubuntu:15.04 You can use virt-sandbox-image download docker://dan@123456:index.docker.io/ubuntu?tag=15.04 The only mandatory part is the source prefix and image name, so that can shorten to just virt-sandbox-image download docker:///ubuntu to pull down the latest ubuntu image, from the default registry using no authentication. --- libvirt-sandbox/image/cli.py | 73 +++++-------- libvirt-sandbox/image/sources/DockerSource.py | 146 ++++++++++++++------------ libvirt-sandbox/image/sources/Source.py | 33 +++--- libvirt-sandbox/image/template.py | 109 +++++++++++++++++++ 4 files changed, 232 insertions(+), 129 deletions(-) create mode 100644 libvirt-sandbox/image/template.py diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py index c8b5ba5..dd109a8 100755 --- a/libvirt-sandbox/image/cli.py +++ b/libvirt-sandbox/image/cli.py @@ -3,7 +3,7 @@ # Authors: Daniel P. Berrange <berrange(a)redhat.com&gt; # Eren Yagdiran <erenyagdiran(a)gmail.com&gt; # -# Copyright (C) 2013 Red Hat, Inc. +# Copyright (C) 2013-2015 Red Hat, Inc. # Copyright (C) 2015 Universitat Politècnica de Catalunya. # # This program is free software; you can redistribute it and/or modify @@ -34,6 +34,8 @@ import subprocess import random import string +from libvirt_sandbox.image import template + if os.geteuid() == 0: default_template_dir = "/var/lib/libvirt/templates" default_image_dir = "/var/lib/libvirt/images" @@ -44,15 +46,6 @@ else: debug = False verbose = False -import importlib -def dynamic_source_loader(name): - name = name[0].upper() + name[1:] - modname = "libvirt_sandbox.image.sources." + name + "Source" - mod = importlib.import_module(modname) - classname = name + "Source" - classimpl = getattr(mod, classname) - return classimpl() - gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") gettext.textdomain("libvirt-sandbox") try: @@ -73,11 +66,10 @@ def info(msg): def download(args): try: - dynamic_source_loader(args.source).download_template(templatename=args.template, - templatedir=args.template_dir, - registry=args.registry, - username=args.username, - password=args.password) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.download_template(template=tmpl, + templatedir=args.template_dir) except IOError,e: print "Source %s cannot be found in given path" %args.source except Exception,e: @@ -85,17 +77,21 @@ def download(args): def delete(args): try: - dynamic_source_loader(args.source).delete_template(templatename=args.template, - templatedir=args.template_dir) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.delete_template(template=tmpl, + templatedir=args.template_dir) except Exception,e: print "Delete Error %s", str(e) def create(args): try: - dynamic_source_loader(args.source).create_template(templatename=args.template, - templatedir=args.template_dir, - connect=args.connect, - format=args.format) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.create_template(template=tmpl, + templatedir=args.template_dir, + connect=args.connect, + format=args.format) except Exception,e: print "Create Error %s" % str(e) @@ -104,19 +100,22 @@ def run(args): global image_dir if args.connect is not None: check_connect(args.connect) - source = dynamic_source_loader(args.source) + + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + name = args.name if name is None: randomid = ''.join(random.choice(string.lowercase) for i in range(10)) - name = args.template + ":" + randomid + name = tmpl.path[1:] + ":" + randomid - diskfile = source.get_disk(templatename=args.template, + diskfile = source.get_disk(template=tmpl, templatedir=args.template_dir, imagedir=args.image_dir, sandboxname=name) format = "qcow2" - commandToRun = source.get_command(args.template, args.template_dir, args.args) + commandToRun = source.get_command(tmpl, args.template_dir, args.args) if len(commandToRun) == 0: commandToRun = ["/bin/sh"] cmd = ['virt-sandbox', '--name', name] @@ -130,7 +129,7 @@ def run(args): params.append('-N') params.append(networkArgs) - allEnvs = source.get_env(args.template, args.template_dir) + allEnvs = source.get_env(tmpl, args.template_dir) envArgs = args.env if envArgs is not None: allEnvs = allEnvs + envArgs @@ -143,7 +142,7 @@ def run(args): else: pass - allVolumes = source.get_volumes(args.template, args.template_dir) + allVolumes = source.get_volumes(tmpl, args.template_dir) volumeArgs = args.volume if volumeArgs is not None: allVolumes = allVolumes + volumeArgs @@ -172,7 +171,7 @@ def run(args): def requires_template(parser): parser.add_argument("template", - help=_("name of the template")) + help=_("URI of the template")) def requires_name(parser): parser.add_argument("-n","--name", @@ -184,23 +183,10 @@ def check_connect(connectstr): raise ValueError("URI '%s' is not supported by virt-sandbox-image" % connectstr) return True -def requires_source(parser): - parser.add_argument("-s","--source", - default="docker", - help=_("name of the template")) - def requires_connect(parser): parser.add_argument("-c","--connect", help=_("Connect string for libvirt")) -def requires_auth_conn(parser): - parser.add_argument("-r","--registry", - help=_("Url of the custom registry")) - parser.add_argument("-u","--username", - help=_("Username for the custom registry")) - parser.add_argument("-p","--password", - help=_("Password for the custom registry")) - def requires_template_dir(parser): global default_template_dir parser.add_argument("-t","--template-dir", @@ -217,8 +203,6 @@ def gen_download_args(subparser): parser = subparser.add_parser("download", help=_("Download template data")) requires_template(parser) - requires_source(parser) - requires_auth_conn(parser) requires_template_dir(parser) parser.set_defaults(func=download) @@ -226,7 +210,6 @@ def gen_delete_args(subparser): parser = subparser.add_parser("delete", help=_("Delete template data")) requires_template(parser) - requires_source(parser) requires_template_dir(parser) parser.set_defaults(func=delete) @@ -234,7 +217,6 @@ def gen_create_args(subparser): parser = subparser.add_parser("create", help=_("Create image from template data")) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) parser.add_argument("-f","--format", @@ -247,7 +229,6 @@ def gen_run_args(subparser): help=_("Run an already built image")) requires_name(parser) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) requires_image_dir(parser) diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py index ff950ce..f367c8f 100644 --- a/libvirt-sandbox/image/sources/DockerSource.py +++ b/libvirt-sandbox/image/sources/DockerSource.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -29,6 +30,8 @@ import os import subprocess import shutil import collections +import urlparse + class DockerConfParser(): @@ -55,12 +58,6 @@ class DockerConfParser(): class DockerSource(Source): - www_auth_username = None - www_auth_password = None - - def __init__(self): - self.default_index_server = "index.docker.io" - def _check_cert_validate(self): major = sys.version_info.major SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" @@ -70,43 +67,38 @@ class DockerSource(Source): if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion): sys.stderr.write(SSL_WARNING) - def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): - if registry is None: - registry = self.default_index_server - - if username is not None: - self.www_auth_username = username - self.www_auth_password = password - + def download_template(self, template, templatedir): self._check_cert_validate() - tag = "latest" - offset = templatename.find(':') - if offset != -1: - tag = templatename[offset + 1:] - templatename = templatename[0:offset] + try: - (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images", - {"X-Docker-Token": "true"}) + (data, res) = self._get_json(template, + None, + "/v1/repositories" + template.path + "/images", + {"X-Docker-Token": "true"}) except urllib2.HTTPError, e: - raise ValueError(["Image '%s' does not exist" % templatename]) + raise ValueError(["Image '%s' does not exist" % template]) registryendpoint = res.info().getheader('X-Docker-Endpoints') token = res.info().getheader('X-Docker-Token') checksums = {} for layer in data: pass - (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags", - { "Authorization": "Token " + token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/repositories" + template.path + "/tags", + { "Authorization": "Token " + token }) cookie = res.info().getheader('Set-Cookie') + tag = template.params.get("tag", "latest") if not tag in data: - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)]) + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, template)]) imagetagid = data[tag] - (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry", - { "Authorization": "Token "+token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/images/" + imagetagid + "/ancestry", + { "Authorization": "Token "+ token }) if data[0] != imagetagid: raise ValueError(["Expected first layer id '%s' to match image id '%s'", @@ -126,8 +118,11 @@ class DockerSource(Source): datafile = layerdir + "/template.tar.gz" if not os.path.exists(jsonfile) or not os.path.exists(datafile): - res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json", - { "Authorization": "Token " + token }, jsonfile) + res = self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/json", + { "Authorization": "Token " + token }, + jsonfile) createdFiles.append(jsonfile) layersize = int(res.info().getheader("Content-Length")) @@ -136,12 +131,15 @@ class DockerSource(Source): if layerid in checksums: datacsum = checksums[layerid] - self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer", - { "Authorization": "Token "+token }, datafile, datacsum, layersize) + self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/layer", + { "Authorization": "Token "+token }, + datafile, datacsum, layersize) createdFiles.append(datafile) index = { - "name": templatename, + "name": template.path, } indexfile = templatedir + "/" + imagetagid + "/index.json" @@ -160,9 +158,11 @@ class DockerSource(Source): shutil.rmtree(d) except: pass - def _save_data(self,server, path, headers, dest, checksum=None, datalen=None): + + def _save_data(self, template, server, path, headers, + dest, checksum=None, datalen=None): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers) csum = None if checksum is not None: @@ -201,8 +201,22 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise - def _get_url(self,server, path, headers): - url = "https://" + server + path + def _get_url(self, template, server, path, headers): + if template.protocol is None: + protocol = "https" + else: + protocol = template.protocol + + if server is None: + if template.hostname is None: + server = "index.docker.io" + else: + if template.port is not None: + server = template.hostname + ":" + template.port + else: + server = template.hostname + + url = urlparse.urlunparse((protocol, server, path, None, None, None)) debug("Fetching %s..." % url) req = urllib2.Request(url=url) @@ -212,16 +226,18 @@ class DockerSource(Source): req.add_header(h, headers[h]) #www Auth header starts - if self.www_auth_username is not None: - base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '') + if template.username and template.password: + base64string = base64.encodestring( + '%s:%s' % (template.username, + template.password)).replace('\n', '') req.add_header("Authorization", "Basic %s" % base64string) #www Auth header finish return urllib2.urlopen(req) - def _get_json(self,server, path, headers): + def _get_json(self, template, server, path, headers): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers) data = json.loads(res.read()) debug("OK\n") return (data, res) @@ -229,11 +245,11 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise - def create_template(self, templatename, templatedir, connect=None, format=None): + def create_template(self, template, templatedir, connect=None, format=None): if format is None: format = self.default_disk_format self._check_disk_format(format) - imagelist = self._get_image_list(templatename,templatedir) + imagelist = self._get_image_list(template, templatedir) imagelist.reverse() parentImage = None @@ -260,7 +276,7 @@ class DockerSource(Source): if not format in supportedFormats: raise ValueError(["Unsupported image format %s" % format]) - def _get_image_list(self,templatename,destdir): + def _get_image_list(self, template, destdir): imageparent = {} imagenames = {} imagedirs = os.listdir(destdir) @@ -273,13 +289,13 @@ class DockerSource(Source): jsonfile = destdir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) - parent = template.get("parent",None) + data = json.load(f) + parent = data.get("parent",None) if parent: imageparent[imagetagid] = parent - if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) - imagetagid = imagenames[templatename] + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path]) + imagetagid = imagenames[template.path] imagelist = [] while imagetagid != None: imagelist.append(imagetagid) @@ -318,7 +334,7 @@ class DockerSource(Source): cmd = cmd + params subprocess.call(cmd) - def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): imageusage = {} imageparent = {} imagenames = {} @@ -332,9 +348,9 @@ class DockerSource(Source): jsonfile = templatedir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) + data = json.load(f) - parent = template.get("parent",None) + parent = data.get("parent",None) if parent: if parent not in imageusage: imageusage[parent] = [] @@ -342,10 +358,10 @@ class DockerSource(Source): imageparent[imagetagid] = parent - if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path]) - imagetagid = imagenames[templatename] + imagetagid = imagenames[template.path] while imagetagid != None: debug("Remove %s\n" % imagetagid) parent = imageparent.get(imagetagid,None) @@ -368,15 +384,15 @@ class DockerSource(Source): parent = None imagetagid = parent - def _get_template_data(self, templatename, templatedir): - imageList = self._get_image_list(templatename, templatedir) + def _get_template_data(self, template, templatedir): + imageList = self._get_image_list(template, templatedir) toplayer = imageList[0] diskfile = templatedir + "/" + toplayer + "/template.qcow2" configfile = templatedir + "/" + toplayer + "/template.json" return configfile, diskfile - def get_disk(self,templatename, templatedir, imagedir, sandboxname): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_disk(self, template, templatedir, imagedir, sandboxname): + configfile, diskfile = self._get_template_data(template, templatedir) tempfile = imagedir + "/" + sandboxname + ".qcow2" if not os.path.exists(imagedir): os.makedirs(imagedir) @@ -387,8 +403,8 @@ class DockerSource(Source): subprocess.call(cmd) return tempfile - def get_command(self, templatename, templatedir, userargs): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_command(self, template, templatedir, userargs): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) cmd = configParser.getCommand() entrypoint = configParser.getEntrypoint() @@ -401,13 +417,13 @@ class DockerSource(Source): else: return entrypoint + cmd - def get_env(self, templatename, templatedir): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_env(self, template, templatedir): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) return configParser.getEnvs() - def get_volumes(self, templatename, templatedir): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_volumes(self, template, templatedir): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) return configParser.getVolumes() diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py index f1dd3e7..bb69682 100644 --- a/libvirt-sandbox/image/sources/Source.py +++ b/libvirt-sandbox/image/sources/Source.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -33,14 +34,10 @@ class Source(): pass @abstractmethod - def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): + def download_template(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template - :param registry: optional hostname of image registry server - :param username: optional username to authenticate against registry server - :param password: optional password to authenticate against registry server Download a template from the registry, storing it in the local filesystem @@ -48,10 +45,10 @@ class Source(): pass @abstractmethod - def create_template(self, templatename, templatedir, + def create_template(self, template, templatedir, connect=None, format=None): """ - :param templatename: name of the template image to create + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template :param connect: libvirt connection URI :param format: disk image format @@ -63,9 +60,9 @@ class Source(): pass @abstractmethod - def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): """ - :param templatename: name of the template image to delete + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path from which to delete template Delete all local files associated with the template @@ -73,9 +70,9 @@ class Source(): pass @abstractmethod - def get_command(self, templatename, templatedir, userargs): + def get_command(self, template, templatedir, userargs): """ - :param templatename: name of the template image to query + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which templates are stored :param userargs: user specified arguments to run @@ -85,9 +82,9 @@ class Source(): pass @abstractmethod - def get_disk(self,templatename, templatedir, imagedir, sandboxname): + def get_disk(self, template, templatedir, imagedir, sandboxname): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template :param imagedir: local directory in which to storage disk image @@ -97,9 +94,9 @@ class Source(): pass @abstractmethod - def get_env(self,templatename, templatedir): + def get_env(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template Get the dict of environment variables to set @@ -107,9 +104,9 @@ class Source(): pass @abstractmethod - def get_volumes(self,templatename, templatedir): + def get_volumes(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template Get the list of volumes associated with the template diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py new file mode 100644 index 0000000..9021242 --- /dev/null +++ b/libvirt-sandbox/image/template.py @@ -0,0 +1,109 @@ +# +# -*- coding: utf-8 -*- +# Authors: Daniel P. Berrange <berrange(a)redhat.com&gt; +# +# Copyright (C) 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# + +import urlparse +import importlib + +class Template(object): + + def __init__(self, + source, protocol, + hostname, port, + username, password, + path, params): + """ + :param source: template source name + :param protocol: network transport protocol or None + :param hostname: registry hostname or None + :param port: registry port or None + :param username: username or None + :param password: password or None + :param path: template path identifier + :param params: template parameters + + docker:///ubuntu + + docker+https://index.docker.io/ubuntu?tag=latest + """ + + self.source = source + self.protocol = protocol + self.hostname = hostname + self.port = port + self.username = username + self.password = password + self.path = path + self.params = params + if self.params is None: + self.params = {} + + def get_source_impl(self): + mod = importlib.import_module( + "libvirt_sandbox.image.sources." + + self.source.capitalize() + "Source") + classname = self.source.capitalize() + "Source" + classimpl = getattr(mod, classname) + return classimpl() + + def __repr__(self): + if self.protocol is not None: + scheme = self.source + "+" + self.protocol + else: + scheme = self.source + if self.hostname: + if self.port: + netloc = self.hostname + ":" + self.port + else: + netloc = self.hostname + + if self.username: + if self.password: + auth = self.username + ":" + self.password + else: + auth = self.username + netloc = auth + "@" + netloc + else: + netloc = None + + query = "&".join([key + "=" + self.params[key] for key in self.params.keys()]) + return urlparse.urlunparse((scheme, netloc, self.path, None, query, None)) + + @classmethod + def from_uri(klass, uri): + o = urlparse.urlparse(uri) + + idx = o.scheme.find("+") + if idx == -1: + source = o.scheme + protocol = None + else: + source = o.scheme[0:idx] + protocol = o.schema[idx + 1:] + + query = {} + for param in o.query.split("&"): + (key, val) = param.split("=") + query[key] = val + return klass(source, protocol, + o.hostname, o.port, + o.username, o.password, + o.path, query) + -- 2.4.3