On Mon, 2015-09-21 at 15:45 +0100, Daniel P. Berrange wrote:

Currently the CLI syntax is somewhat docker specific requiring inclusion of --registry arg to identify the docker download server. Other app containers have a notion of download server, but don't separate it from the template name.

This patch removes that docker-ism by changing to use a URI for identifying the template image. So instead of

virt-sandbox-image download \ --source docker --registry index.docker.io --username dan --password 123456 ubuntu:15.04

You can use

virt-sandbox-image download docker://dan:123456@index.docker.io/ubuntu?tag=15.04

The only mandatory part is the source prefix and image name, so that can shorten to just

virt-sandbox-image download docker:///ubuntu

to pull down the latest ubuntu image, from the default registry using no authentication. ---

Changed in v2:

- Rebase against master, instead of (unpushed) docker volume patch

libvirt-sandbox/image/cli.py | 71 +++++-------- libvirt-sandbox/image/sources/DockerSource.py | 142 ++++++++++++++------------ libvirt-sandbox/image/sources/Source.py | 29 +++--- libvirt-sandbox/image/template.py | 110 ++++++++++++++++++++

Missing change in libvirt-sandbox/image/Makefile.am to add template.py. As is that file isn't installed. I'm also just realizing that we didn't add Eren't commit for the virt-sandbox-image man page. Adding it later is fine, but we need to keep that on our radar.

4 files changed, 228 insertions(+), 124 deletions(-) create mode 100644 libvirt-sandbox/image/template.py

diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py index 1718cc5..4d02289 100755 --- a/libvirt-sandbox/image/cli.py +++ b/libvirt-sandbox/image/cli.py @@ -3,7 +3,7 @@ # Authors: Daniel P. Berrange <berrange@redhat.com> # Eren Yagdiran <erenyagdiran@gmail.com> # -# Copyright (C) 2013 Red Hat, Inc. +# Copyright (C) 2013-2015 Red Hat, Inc. # Copyright (C) 2015 Universitat Politècnica de Catalunya. # # This program is free software; you can redistribute it and/or modify @@ -34,6 +34,8 @@ import subprocess import random import string

+from libvirt_sandbox.image import template + if os.geteuid() == 0: default_template_dir = "/var/lib/libvirt/templates" default_image_dir = "/var/lib/libvirt/images" @@ -44,15 +46,6 @@ else: debug = False verbose = False

-import importlib -def dynamic_source_loader(name): - name = name[0].upper() + name[1:] - modname = "libvirt_sandbox.image.sources." + name + "Source" - mod = importlib.import_module(modname) - classname = name + "Source" - classimpl = getattr(mod, classname) - return classimpl() - gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") gettext.textdomain("libvirt-sandbox") try: @@ -73,11 +66,10 @@ def info(msg):

def download(args): try: - dynamic_source_loader(args.source).download_template(templatename=args.template, - templatedir=args.template_dir, - registry=args.registry, - username=args.username, - password=args.password) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.download_template(template=tmpl, + templatedir=args.template_dir) except IOError,e: print "Source %s cannot be found in given path" %args.source except Exception,e: @@ -85,17 +77,21 @@ def download(args):

def delete(args): try: - dynamic_source_loader(args.source).delete_template(templatename=args.template, - templatedir=args.template_dir) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.delete_template(template=tmpl, + templatedir=args.template_dir) except Exception,e: print "Delete Error %s", str(e)

def create(args): try: - dynamic_source_loader(args.source).create_template(templatename=args.template, - templatedir=args.template_dir, - connect=args.connect, - format=args.format) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.create_template(template=tmpl, + templatedir=args.template_dir, + connect=args.connect, + format=args.format) except Exception,e: print "Create Error %s" % str(e)

@@ -103,19 +99,22 @@ def run(args): try: if args.connect is not None: check_connect(args.connect) - source = dynamic_source_loader(args.source) + + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + name = args.name if name is None: randomid = ''.join(random.choice(string.lowercase) for i in range(10)) - name = args.template + ":" + randomid + name = tmpl.path[1:] + ":" + randomid

- diskfile = source.get_disk(templatename=args.template, + diskfile = source.get_disk(template=tmpl, templatedir=args.template_dir, imagedir=args.image_dir, sandboxname=name)

format = "qcow2" - commandToRun = source.get_command(args.template, args.template_dir, args.args) + commandToRun = source.get_command(tmpl, args.template_dir, args.args) if len(commandToRun) == 0: commandToRun = ["/bin/sh"] cmd = ['virt-sandbox', '--name', name] @@ -129,7 +128,7 @@ def run(args): params.append('-N') params.append(networkArgs)

- allEnvs = source.get_env(args.template, args.template_dir) + allEnvs = source.get_env(tmpl, args.template_dir) envArgs = args.env if envArgs is not None: allEnvs = allEnvs + envArgs @@ -151,7 +150,7 @@ def run(args):

def requires_template(parser): parser.add_argument("template", - help=_("name of the template")) + help=_("URI of the template"))

Shouldn't we provide some examples here? As those URIs can't be invented we need to give the user some chances to discover them without having to read our code ;)

def requires_name(parser): parser.add_argument("-n","--name", @@ -163,23 +162,10 @@ def check_connect(connectstr): raise ValueError("URI '%s' is not supported by virt-sandbox-image" % connectstr) return True

-def requires_source(parser): - parser.add_argument("-s","--source", - default="docker", - help=_("name of the template")) - def requires_connect(parser): parser.add_argument("-c","--connect", help=_("Connect string for libvirt"))

-def requires_auth_conn(parser): - parser.add_argument("-r","--registry", - help=_("Url of the custom registry")) - parser.add_argument("-u","--username", - help=_("Username for the custom registry")) - parser.add_argument("-p","--password", - help=_("Password for the custom registry")) - def requires_template_dir(parser): global default_template_dir parser.add_argument("-t","--template-dir", @@ -196,8 +182,6 @@ def gen_download_args(subparser): parser = subparser.add_parser("download", help=_("Download template data")) requires_template(parser) - requires_source(parser) - requires_auth_conn(parser) requires_template_dir(parser) parser.set_defaults(func=download)

@@ -205,7 +189,6 @@ def gen_delete_args(subparser): parser = subparser.add_parser("delete", help=_("Delete template data")) requires_template(parser) - requires_source(parser) requires_template_dir(parser) parser.set_defaults(func=delete)

@@ -213,7 +196,6 @@ def gen_create_args(subparser): parser = subparser.add_parser("create", help=_("Create image from template data")) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) parser.add_argument("-f","--format", @@ -226,7 +208,6 @@ def gen_run_args(subparser): help=_("Run an already built image")) requires_name(parser) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) requires_image_dir(parser) diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py index c374a0c..10f8537 100644 --- a/libvirt-sandbox/image/sources/DockerSource.py +++ b/libvirt-sandbox/image/sources/DockerSource.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -28,6 +29,8 @@ import traceback import os import subprocess import shutil +import urlparse +

class DockerConfParser():

@@ -47,12 +50,6 @@ class DockerConfParser():

class DockerSource(Source):

- www_auth_username = None - www_auth_password = None - - def __init__(self): - self.default_index_server = "index.docker.io" - def _check_cert_validate(self): major = sys.version_info.major SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" @@ -62,43 +59,38 @@ class DockerSource(Source): if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion): sys.stderr.write(SSL_WARNING)

- def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): - if registry is None: - registry = self.default_index_server - - if username is not None: - self.www_auth_username = username - self.www_auth_password = password - + def download_template(self, template, templatedir): self._check_cert_validate() - tag = "latest" - offset = templatename.find(':') - if offset != -1: - tag = templatename[offset + 1:] - templatename = templatename[0:offset] + try: - (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images", - {"X-Docker-Token": "true"}) + (data, res) = self._get_json(template, + None, + "/v1/repositories" + template.path + "/images", + {"X-Docker-Token": "true"}) except urllib2.HTTPError, e: - raise ValueError(["Image '%s' does not exist" % templatename]) + raise ValueError(["Image '%s' does not exist" % template])

registryendpoint = res.info().getheader('X-Docker-Endpoints') token = res.info().getheader('X-Docker-Token') checksums = {} for layer in data: pass - (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags", - { "Authorization": "Token " + token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/repositories" + template.path + "/tags", + { "Authorization": "Token " + token })

cookie = res.info().getheader('Set-Cookie')

+ tag = template.params.get("tag", "latest") if not tag in data: - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)]) + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, template)]) imagetagid = data[tag]

- (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry", - { "Authorization": "Token "+token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/images/" + imagetagid + "/ancestry", + { "Authorization": "Token "+ token })

if data[0] != imagetagid: raise ValueError(["Expected first layer id '%s' to match image id '%s'", @@ -118,8 +110,11 @@ class DockerSource(Source): datafile = layerdir + "/template.tar.gz"

if not os.path.exists(jsonfile) or not os.path.exists(datafile): - res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json", - { "Authorization": "Token " + token }, jsonfile) + res = self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/json", + { "Authorization": "Token " + token }, + jsonfile) createdFiles.append(jsonfile)

layersize = int(res.info().getheader("Content-Length")) @@ -128,12 +123,15 @@ class DockerSource(Source): if layerid in checksums: datacsum = checksums[layerid]

- self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer", - { "Authorization": "Token "+token }, datafile, datacsum, layersize) + self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/layer", + { "Authorization": "Token "+token }, + datafile, datacsum, layersize) createdFiles.append(datafile)

index = { - "name": templatename, + "name": template.path, }

indexfile = templatedir + "/" + imagetagid + "/index.json" @@ -152,9 +150,11 @@ class DockerSource(Source): shutil.rmtree(d) except: pass - def _save_data(self,server, path, headers, dest, checksum=None, datalen=None): + + def _save_data(self, template, server, path, headers, + dest, checksum=None, datalen=None): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers)

csum = None if checksum is not None: @@ -193,8 +193,22 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise

- def _get_url(self,server, path, headers): - url = "https://" + server + path + def _get_url(self, template, server, path, headers): + if template.protocol is None: + protocol = "https" + else: + protocol = template.protocol + + if server is None: + if template.hostname is None: + server = "index.docker.io" + else: + if template.port is not None: + server = template.hostname + ":" + template.port + else: + server = template.hostname + + url = urlparse.urlunparse((protocol, server, path, None, None, None)) debug("Fetching %s..." % url)

req = urllib2.Request(url=url) @@ -204,16 +218,18 @@ class DockerSource(Source): req.add_header(h, headers[h])

#www Auth header starts - if self.www_auth_username is not None: - base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '') + if template.username and template.password: + base64string = base64.encodestring( + '%s:%s' % (template.username, + template.password)).replace('\n', '') req.add_header("Authorization", "Basic %s" % base64string) #www Auth header finish

return urllib2.urlopen(req)

- def _get_json(self,server, path, headers): + def _get_json(self, template, server, path, headers): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers) data = json.loads(res.read()) debug("OK\n") return (data, res) @@ -221,11 +237,11 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise

- def create_template(self, templatename, templatedir, connect=None, format=None): + def create_template(self, template, templatedir, connect=None, format=None): if format is None: format = self.default_disk_format self._check_disk_format(format) - imagelist = self._get_image_list(templatename,templatedir) + imagelist = self._get_image_list(template, templatedir) imagelist.reverse()

parentImage = None @@ -252,7 +268,7 @@ class DockerSource(Source): if not format in supportedFormats: raise ValueError(["Unsupported image format %s" % format])

- def _get_image_list(self,templatename,destdir): + def _get_image_list(self, template, destdir): imageparent = {} imagenames = {} imagedirs = os.listdir(destdir) @@ -265,13 +281,13 @@ class DockerSource(Source): jsonfile = destdir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) - parent = template.get("parent",None) + data = json.load(f) + parent = data.get("parent",None) if parent: imageparent[imagetagid] = parent - if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) - imagetagid = imagenames[templatename] + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path]) + imagetagid = imagenames[template.path] imagelist = [] while imagetagid != None: imagelist.append(imagetagid) @@ -310,7 +326,7 @@ class DockerSource(Source): cmd = cmd + params subprocess.call(cmd)

- def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): imageusage = {} imageparent = {} imagenames = {} @@ -324,9 +340,9 @@ class DockerSource(Source): jsonfile = templatedir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) + data = json.load(f)

- parent = template.get("parent",None) + parent = data.get("parent",None) if parent: if parent not in imageusage: imageusage[parent] = [] @@ -334,10 +350,10 @@ class DockerSource(Source): imageparent[imagetagid] = parent

- if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path])

- imagetagid = imagenames[templatename] + imagetagid = imagenames[template.path] while imagetagid != None: debug("Remove %s\n" % imagetagid) parent = imageparent.get(imagetagid,None) @@ -360,15 +376,15 @@ class DockerSource(Source): parent = None imagetagid = parent

- def _get_template_data(self, templatename, templatedir): - imageList = self._get_image_list(templatename, templatedir) + def _get_template_data(self, template, templatedir): + imageList = self._get_image_list(template, templatedir) toplayer = imageList[0] diskfile = templatedir + "/" + toplayer + "/template.qcow2" configfile = templatedir + "/" + toplayer + "/template.json" return configfile, diskfile

- def get_disk(self,templatename, templatedir, imagedir, sandboxname): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_disk(self, template, templatedir, imagedir, sandboxname): + configfile, diskfile = self._get_template_data(template, templatedir) tempfile = imagedir + "/" + sandboxname + ".qcow2" if not os.path.exists(imagedir): os.makedirs(imagedir) @@ -379,8 +395,8 @@ class DockerSource(Source): subprocess.call(cmd) return tempfile

- def get_command(self, templatename, templatedir, userargs): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_command(self, template, templatedir, userargs): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) cmd = configParser.getCommand() entrypoint = configParser.getEntrypoint() @@ -393,8 +409,8 @@ class DockerSource(Source): else: return entrypoint + cmd

- def get_env(self, templatename, templatedir): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_env(self, template, templatedir): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) return configParser.getEnvs()

diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py index 8a21f90..597a7fb 100644 --- a/libvirt-sandbox/image/sources/Source.py +++ b/libvirt-sandbox/image/sources/Source.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -33,14 +34,10 @@ class Source(): pass

@abstractmethod - def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): + def download_template(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template - :param registry: optional hostname of image registry server - :param username: optional username to authenticate against registry server - :param password: optional password to authenticate against registry server

Download a template from the registry, storing it in the local filesystem @@ -48,10 +45,10 @@ class Source(): pass

@abstractmethod - def create_template(self, templatename, templatedir, + def create_template(self, template, templatedir, connect=None, format=None): """ - :param templatename: name of the template image to create + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template :param connect: libvirt connection URI :param format: disk image format @@ -63,9 +60,9 @@ class Source(): pass

@abstractmethod - def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): """ - :param templatename: name of the template image to delete + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path from which to delete template

Delete all local files associated with the template @@ -73,9 +70,9 @@ class Source(): pass

@abstractmethod - def get_command(self, templatename, templatedir, userargs): + def get_command(self, template, templatedir, userargs): """ - :param templatename: name of the template image to query + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which templates are stored :param userargs: user specified arguments to run

@@ -85,9 +82,9 @@ class Source(): pass

@abstractmethod - def get_disk(self,templatename, templatedir, imagedir, sandboxname): + def get_disk(self, template, templatedir, imagedir, sandboxname): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template :param imagedir: local directory in which to storage disk image

@@ -97,9 +94,9 @@ class Source(): pass

@abstractmethod - def get_env(self,templatename, templatedir): + def get_env(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template

Get the dict of environment variables to set diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py new file mode 100644 index 0000000..0ad767b --- /dev/null +++ b/libvirt-sandbox/image/template.py @@ -0,0 +1,110 @@ +# +# -*- coding: utf-8 -*- +# Authors: Daniel P. Berrange <berrange@redhat.com> +# +# Copyright (C) 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# + +import urlparse +import importlib + +class Template(object): + + def __init__(self, + source, protocol, + hostname, port, + username, password, + path, params): + """ + :param source: template source name + :param protocol: network transport protocol or None + :param hostname: registry hostname or None + :param port: registry port or None + :param username: username or None + :param password: password or None + :param path: template path identifier + :param params: template parameters + + docker:///ubuntu + + docker+https://index.docker.io/ubuntu?tag=latest + """ + + self.source = source + self.protocol = protocol + self.hostname = hostname + self.port = port + self.username = username + self.password = password + self.path = path + self.params = params + if self.params is None: + self.params = {} + + def get_source_impl(self): + mod = importlib.import_module( + "libvirt_sandbox.image.sources." + + self.source.capitalize() + "Source") + classname = self.source.capitalize() + "Source" + classimpl = getattr(mod, classname) + return classimpl() + + def __repr__(self): + if self.protocol is not None: + scheme = self.source + "+" + self.protocol + else: + scheme = self.source + if self.hostname: + if self.port: + netloc = self.hostname + ":" + self.port + else: + netloc = self.hostname + + if self.username: + if self.password: + auth = self.username + ":" + self.password + else: + auth = self.username + netloc = auth + "@" + netloc + else: + netloc = None + + query = "&".join([key + "=" + self.params[key] for key in self.params.keys()]) + return urlparse.urlunparse((scheme, netloc, self.path, None, query, None)) + + @classmethod + def from_uri(klass, uri): + o = urlparse.urlparse(uri) + + idx = o.scheme.find("+") + if idx == -1: + source = o.scheme + protocol = None + else: + source = o.scheme[0:idx] + protocol = o.schema[idx + 1:] + + query = {} + if o.query is not None and o.query != "": + for param in o.query.split("&"): + (key, val) = param.split("=") + query[key] = val + return klass(source, protocol, + o.hostname, o.port, + o.username, o.password, + o.path, query) +

git complains about the empty line here. ACK, with the help improvement + Makefile.am fix. -- Cedric

On Mon, 2015-09-21 at 15:45 +0100, Daniel P. Berrange wrote:

Currently the CLI syntax is somewhat docker specific requiring inclusion of --registry arg to identify the docker download server. Other app containers have a notion of download server, but don't separate it from the template name.

This patch removes that docker-ism by changing to use a URI for identifying the template image. So instead of

virt-sandbox-image download \ --source docker --registry index.docker.io --username dan --password 123456 ubuntu:15.04

You can use

virt-sandbox-image download docker://dan:123456@index.docker.io/ubuntu?tag=15.04

The only mandatory part is the source prefix and image name, so that can shorten to just

virt-sandbox-image download docker:///ubuntu

to pull down the latest ubuntu image, from the default registry using no authentication. ---

Changed in v2:

- Rebase against master, instead of (unpushed) docker volume patch

libvirt-sandbox/image/cli.py | 71 +++++-------- libvirt-sandbox/image/sources/DockerSource.py | 142 ++++++++++++++------------ libvirt-sandbox/image/sources/Source.py | 29 +++--- libvirt-sandbox/image/template.py | 110 ++++++++++++++++++++ 4 files changed, 228 insertions(+), 124 deletions(-) create mode 100644 libvirt-sandbox/image/template.py

diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py index 1718cc5..4d02289 100755 --- a/libvirt-sandbox/image/cli.py +++ b/libvirt-sandbox/image/cli.py @@ -3,7 +3,7 @@ # Authors: Daniel P. Berrange <berrange@redhat.com> # Eren Yagdiran <erenyagdiran@gmail.com> # -# Copyright (C) 2013 Red Hat, Inc. +# Copyright (C) 2013-2015 Red Hat, Inc. # Copyright (C) 2015 Universitat Politècnica de Catalunya. # # This program is free software; you can redistribute it and/or modify @@ -34,6 +34,8 @@ import subprocess import random import string

+from libvirt_sandbox.image import template + if os.geteuid() == 0: default_template_dir = "/var/lib/libvirt/templates" default_image_dir = "/var/lib/libvirt/images" @@ -44,15 +46,6 @@ else: debug = False verbose = False

-import importlib -def dynamic_source_loader(name): - name = name[0].upper() + name[1:] - modname = "libvirt_sandbox.image.sources." + name + "Source" - mod = importlib.import_module(modname) - classname = name + "Source" - classimpl = getattr(mod, classname) - return classimpl() - gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") gettext.textdomain("libvirt-sandbox") try: @@ -73,11 +66,10 @@ def info(msg):

def download(args): try: - dynamic_source_loader(args.source).download_template(templatename=args.template, - templatedir=args.template_dir, - registry=args.registry, - username=args.username, - password=args.password) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.download_template(template=tmpl, + templatedir=args.template_dir) except IOError,e: print "Source %s cannot be found in given path" %args.source except Exception,e: @@ -85,17 +77,21 @@ def download(args):

def delete(args): try: - dynamic_source_loader(args.source).delete_template(templatename=args.template, - templatedir=args.template_dir) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.delete_template(template=tmpl, + templatedir=args.template_dir) except Exception,e: print "Delete Error %s", str(e)

def create(args): try: - dynamic_source_loader(args.source).create_template(templatename=args.template, - templatedir=args.template_dir, - connect=args.connect, - format=args.format) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.create_template(template=tmpl, + templatedir=args.template_dir, + connect=args.connect, + format=args.format) except Exception,e: print "Create Error %s" % str(e)

@@ -103,19 +99,22 @@ def run(args): try: if args.connect is not None: check_connect(args.connect) - source = dynamic_source_loader(args.source) + + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + name = args.name if name is None: randomid = ''.join(random.choice(string.lowercase) for i in range(10)) - name = args.template + ":" + randomid + name = tmpl.path[1:] + ":" + randomid

- diskfile = source.get_disk(templatename=args.template, + diskfile = source.get_disk(template=tmpl, templatedir=args.template_dir, imagedir=args.image_dir, sandboxname=name)

format = "qcow2" - commandToRun = source.get_command(args.template, args.template_dir, args.args) + commandToRun = source.get_command(tmpl, args.template_dir, args.args) if len(commandToRun) == 0: commandToRun = ["/bin/sh"] cmd = ['virt-sandbox', '--name', name] @@ -129,7 +128,7 @@ def run(args): params.append('-N') params.append(networkArgs)

- allEnvs = source.get_env(args.template, args.template_dir) + allEnvs = source.get_env(tmpl, args.template_dir) envArgs = args.env if envArgs is not None: allEnvs = allEnvs + envArgs @@ -151,7 +150,7 @@ def run(args):

def requires_template(parser): parser.add_argument("template", - help=_("name of the template")) + help=_("URI of the template"))

def requires_name(parser): parser.add_argument("-n","--name", @@ -163,23 +162,10 @@ def check_connect(connectstr): raise ValueError("URI '%s' is not supported by virt-sandbox-image" % connectstr) return True

-def requires_source(parser): - parser.add_argument("-s","--source", - default="docker", - help=_("name of the template")) - def requires_connect(parser): parser.add_argument("-c","--connect", help=_("Connect string for libvirt"))

-def requires_auth_conn(parser): - parser.add_argument("-r","--registry", - help=_("Url of the custom registry")) - parser.add_argument("-u","--username", - help=_("Username for the custom registry")) - parser.add_argument("-p","--password", - help=_("Password for the custom registry")) - def requires_template_dir(parser): global default_template_dir parser.add_argument("-t","--template-dir", @@ -196,8 +182,6 @@ def gen_download_args(subparser): parser = subparser.add_parser("download", help=_("Download template data")) requires_template(parser) - requires_source(parser) - requires_auth_conn(parser) requires_template_dir(parser) parser.set_defaults(func=download)

@@ -205,7 +189,6 @@ def gen_delete_args(subparser): parser = subparser.add_parser("delete", help=_("Delete template data")) requires_template(parser) - requires_source(parser) requires_template_dir(parser) parser.set_defaults(func=delete)

@@ -213,7 +196,6 @@ def gen_create_args(subparser): parser = subparser.add_parser("create", help=_("Create image from template data")) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) parser.add_argument("-f","--format", @@ -226,7 +208,6 @@ def gen_run_args(subparser): help=_("Run an already built image")) requires_name(parser) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) requires_image_dir(parser) diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py index c374a0c..10f8537 100644 --- a/libvirt-sandbox/image/sources/DockerSource.py +++ b/libvirt-sandbox/image/sources/DockerSource.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -28,6 +29,8 @@ import traceback import os import subprocess import shutil +import urlparse +

class DockerConfParser():

@@ -47,12 +50,6 @@ class DockerConfParser():

class DockerSource(Source):

- www_auth_username = None - www_auth_password = None - - def __init__(self): - self.default_index_server = "index.docker.io" - def _check_cert_validate(self): major = sys.version_info.major SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" @@ -62,43 +59,38 @@ class DockerSource(Source): if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion): sys.stderr.write(SSL_WARNING)

- def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): - if registry is None: - registry = self.default_index_server - - if username is not None: - self.www_auth_username = username - self.www_auth_password = password - + def download_template(self, template, templatedir): self._check_cert_validate() - tag = "latest" - offset = templatename.find(':') - if offset != -1: - tag = templatename[offset + 1:] - templatename = templatename[0:offset] + try: - (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images", - {"X-Docker-Token": "true"}) + (data, res) = self._get_json(template, + None, + "/v1/repositories" + template.path + "/images", + {"X-Docker-Token": "true"}) except urllib2.HTTPError, e: - raise ValueError(["Image '%s' does not exist" % templatename]) + raise ValueError(["Image '%s' does not exist" % template])

registryendpoint = res.info().getheader('X-Docker-Endpoints') token = res.info().getheader('X-Docker-Token') checksums = {} for layer in data: pass - (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags", - { "Authorization": "Token " + token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/repositories" + template.path + "/tags", + { "Authorization": "Token " + token })

cookie = res.info().getheader('Set-Cookie')

+ tag = template.params.get("tag", "latest") if not tag in data: - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)]) + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, template)]) imagetagid = data[tag]

- (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry", - { "Authorization": "Token "+token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/images/" + imagetagid + "/ancestry", + { "Authorization": "Token "+ token })

if data[0] != imagetagid: raise ValueError(["Expected first layer id '%s' to match image id '%s'", @@ -118,8 +110,11 @@ class DockerSource(Source): datafile = layerdir + "/template.tar.gz"

if not os.path.exists(jsonfile) or not os.path.exists(datafile): - res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json", - { "Authorization": "Token " + token }, jsonfile) + res = self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/json", + { "Authorization": "Token " + token }, + jsonfile) createdFiles.append(jsonfile)

layersize = int(res.info().getheader("Content-Length")) @@ -128,12 +123,15 @@ class DockerSource(Source): if layerid in checksums: datacsum = checksums[layerid]

- self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer", - { "Authorization": "Token "+token }, datafile, datacsum, layersize) + self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/layer", + { "Authorization": "Token "+token }, + datafile, datacsum, layersize) createdFiles.append(datafile)

index = { - "name": templatename, + "name": template.path, }

indexfile = templatedir + "/" + imagetagid + "/index.json" @@ -152,9 +150,11 @@ class DockerSource(Source): shutil.rmtree(d) except: pass - def _save_data(self,server, path, headers, dest, checksum=None, datalen=None): + + def _save_data(self, template, server, path, headers, + dest, checksum=None, datalen=None): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers)

csum = None if checksum is not None: @@ -193,8 +193,22 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise

- def _get_url(self,server, path, headers): - url = "https://" + server + path + def _get_url(self, template, server, path, headers): + if template.protocol is None: + protocol = "https" + else: + protocol = template.protocol + + if server is None: + if template.hostname is None: + server = "index.docker.io" + else: + if template.port is not None: + server = template.hostname + ":" + template.port

This doesn't fly, port is an int, we need to have it in this form: server = "%s:%d" % (template.hostname, template.port)

+ else: + server = template.hostname + + url = urlparse.urlunparse((protocol, server, path, None, None, None)) debug("Fetching %s..." % url)

req = urllib2.Request(url=url) @@ -204,16 +218,18 @@ class DockerSource(Source): req.add_header(h, headers[h])

#www Auth header starts - if self.www_auth_username is not None: - base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '') + if template.username and template.password: + base64string = base64.encodestring( + '%s:%s' % (template.username, + template.password)).replace('\n', '') req.add_header("Authorization", "Basic %s" % base64string) #www Auth header finish

return urllib2.urlopen(req)

- def _get_json(self,server, path, headers): + def _get_json(self, template, server, path, headers): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers) data = json.loads(res.read()) debug("OK\n") return (data, res) @@ -221,11 +237,11 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise

- def create_template(self, templatename, templatedir, connect=None, format=None): + def create_template(self, template, templatedir, connect=None, format=None): if format is None: format = self.default_disk_format self._check_disk_format(format) - imagelist = self._get_image_list(templatename,templatedir) + imagelist = self._get_image_list(template, templatedir) imagelist.reverse()

parentImage = None @@ -252,7 +268,7 @@ class DockerSource(Source): if not format in supportedFormats: raise ValueError(["Unsupported image format %s" % format])

- def _get_image_list(self,templatename,destdir): + def _get_image_list(self, template, destdir): imageparent = {} imagenames = {} imagedirs = os.listdir(destdir) @@ -265,13 +281,13 @@ class DockerSource(Source): jsonfile = destdir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) - parent = template.get("parent",None) + data = json.load(f) + parent = data.get("parent",None) if parent: imageparent[imagetagid] = parent - if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) - imagetagid = imagenames[templatename] + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path]) + imagetagid = imagenames[template.path] imagelist = [] while imagetagid != None: imagelist.append(imagetagid) @@ -310,7 +326,7 @@ class DockerSource(Source): cmd = cmd + params subprocess.call(cmd)

- def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): imageusage = {} imageparent = {} imagenames = {} @@ -324,9 +340,9 @@ class DockerSource(Source): jsonfile = templatedir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) + data = json.load(f)

- parent = template.get("parent",None) + parent = data.get("parent",None) if parent: if parent not in imageusage: imageusage[parent] = [] @@ -334,10 +350,10 @@ class DockerSource(Source): imageparent[imagetagid] = parent

- if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path])

- imagetagid = imagenames[templatename] + imagetagid = imagenames[template.path] while imagetagid != None: debug("Remove %s\n" % imagetagid) parent = imageparent.get(imagetagid,None) @@ -360,15 +376,15 @@ class DockerSource(Source): parent = None imagetagid = parent

- def _get_template_data(self, templatename, templatedir): - imageList = self._get_image_list(templatename, templatedir) + def _get_template_data(self, template, templatedir): + imageList = self._get_image_list(template, templatedir) toplayer = imageList[0] diskfile = templatedir + "/" + toplayer + "/template.qcow2" configfile = templatedir + "/" + toplayer + "/template.json" return configfile, diskfile

- def get_disk(self,templatename, templatedir, imagedir, sandboxname): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_disk(self, template, templatedir, imagedir, sandboxname): + configfile, diskfile = self._get_template_data(template, templatedir) tempfile = imagedir + "/" + sandboxname + ".qcow2" if not os.path.exists(imagedir): os.makedirs(imagedir) @@ -379,8 +395,8 @@ class DockerSource(Source): subprocess.call(cmd) return tempfile

- def get_command(self, templatename, templatedir, userargs): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_command(self, template, templatedir, userargs): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) cmd = configParser.getCommand() entrypoint = configParser.getEntrypoint() @@ -393,8 +409,8 @@ class DockerSource(Source): else: return entrypoint + cmd

- def get_env(self, templatename, templatedir): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_env(self, template, templatedir): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) return configParser.getEnvs()

diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py index 8a21f90..597a7fb 100644 --- a/libvirt-sandbox/image/sources/Source.py +++ b/libvirt-sandbox/image/sources/Source.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -33,14 +34,10 @@ class Source(): pass

@abstractmethod - def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): + def download_template(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template - :param registry: optional hostname of image registry server - :param username: optional username to authenticate against registry server - :param password: optional password to authenticate against registry server

Download a template from the registry, storing it in the local filesystem @@ -48,10 +45,10 @@ class Source(): pass

@abstractmethod - def create_template(self, templatename, templatedir, + def create_template(self, template, templatedir, connect=None, format=None): """ - :param templatename: name of the template image to create + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template :param connect: libvirt connection URI :param format: disk image format @@ -63,9 +60,9 @@ class Source(): pass

@abstractmethod - def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): """ - :param templatename: name of the template image to delete + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path from which to delete template

Delete all local files associated with the template @@ -73,9 +70,9 @@ class Source(): pass

@abstractmethod - def get_command(self, templatename, templatedir, userargs): + def get_command(self, template, templatedir, userargs): """ - :param templatename: name of the template image to query + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which templates are stored :param userargs: user specified arguments to run

@@ -85,9 +82,9 @@ class Source(): pass

@abstractmethod - def get_disk(self,templatename, templatedir, imagedir, sandboxname): + def get_disk(self, template, templatedir, imagedir, sandboxname): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template :param imagedir: local directory in which to storage disk image

@@ -97,9 +94,9 @@ class Source(): pass

@abstractmethod - def get_env(self,templatename, templatedir): + def get_env(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template

Get the dict of environment variables to set diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py new file mode 100644 index 0000000..0ad767b --- /dev/null +++ b/libvirt-sandbox/image/template.py @@ -0,0 +1,110 @@ +# +# -*- coding: utf-8 -*- +# Authors: Daniel P. Berrange <berrange@redhat.com> +# +# Copyright (C) 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# + +import urlparse +import importlib + +class Template(object): + + def __init__(self, + source, protocol, + hostname, port, + username, password, + path, params): + """ + :param source: template source name + :param protocol: network transport protocol or None + :param hostname: registry hostname or None + :param port: registry port or None + :param username: username or None + :param password: password or None + :param path: template path identifier + :param params: template parameters + + docker:///ubuntu + + docker+https://index.docker.io/ubuntu?tag=latest + """ + + self.source = source + self.protocol = protocol + self.hostname = hostname + self.port = port + self.username = username + self.password = password + self.path = path + self.params = params + if self.params is None: + self.params = {} + + def get_source_impl(self): + mod = importlib.import_module( + "libvirt_sandbox.image.sources." + + self.source.capitalize() + "Source") + classname = self.source.capitalize() + "Source" + classimpl = getattr(mod, classname) + return classimpl() + + def __repr__(self): + if self.protocol is not None: + scheme = self.source + "+" + self.protocol + else: + scheme = self.source + if self.hostname: + if self.port: + netloc = self.hostname + ":" + self.port

This doesn't work, python requires to put it this way: netloc = "%s:%d" % (self.hostname, self.port) -- Cedric

+ else: + netloc = self.hostname + + if self.username: + if self.password: + auth = self.username + ":" + self.password + else: + auth = self.username + netloc = auth + "@" + netloc + else: + netloc = None + + query = "&".join([key + "=" + self.params[key] for key in self.params.keys()]) + return urlparse.urlunparse((scheme, netloc, self.path, None, query, None)) + + @classmethod + def from_uri(klass, uri): + o = urlparse.urlparse(uri) + + idx = o.scheme.find("+") + if idx == -1: + source = o.scheme + protocol = None + else: + source = o.scheme[0:idx] + protocol = o.schema[idx + 1:]

s/schema/scheme/

+ + query = {} + if o.query is not None and o.query != "": + for param in o.query.split("&"): + (key, val) = param.split("=") + query[key] = val + return klass(source, protocol, + o.hostname, o.port, + o.username, o.password, + o.path, query) +