Currently the CLI syntax is somewhat docker specific requiring
inclusion of --registry arg to identify the docker download
server. Other app containers have a notion of download server,
but don't separate it from the template name.
This patch removes that docker-ism by changing to use a URI
for identifying the template image. So instead of
virt-sandbox-image download \
--source docker --registry index.docker.io
--username dan --password 123456 ubuntu:15.04
You can use
virt-sandbox-image download docker://dan:123456@index.docker.io/ubuntu?tag=15.04
The only mandatory part is the source prefix and image name, so
that can shorten to just
virt-sandbox-image download docker:///ubuntu
to pull down the latest ubuntu image, from the default registry
using no authentication.
---
Changed in v2:
- Rebase against master, instead of (unpushed) docker volume patch
libvirt-sandbox/image/cli.py | 71 +++++--------
libvirt-sandbox/image/sources/DockerSource.py | 142 ++++++++++++++------------
libvirt-sandbox/image/sources/Source.py | 29 +++---
libvirt-sandbox/image/template.py | 110 ++++++++++++++++++++
4 files changed, 228 insertions(+), 124 deletions(-)
create mode 100644 libvirt-sandbox/image/template.py
diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py
index 1718cc5..4d02289 100755
--- a/libvirt-sandbox/image/cli.py
+++ b/libvirt-sandbox/image/cli.py
@@ -3,7 +3,7 @@
# Authors: Daniel P. Berrange <berrange(a)redhat.com>
# Eren Yagdiran <erenyagdiran(a)gmail.com>
#
-# Copyright (C) 2013 Red Hat, Inc.
+# Copyright (C) 2013-2015 Red Hat, Inc.
# Copyright (C) 2015 Universitat Politècnica de Catalunya.
#
# This program is free software; you can redistribute it and/or modify
@@ -34,6 +34,8 @@ import subprocess
import random
import string
+from libvirt_sandbox.image import template
+
if os.geteuid() == 0:
default_template_dir = "/var/lib/libvirt/templates"
default_image_dir = "/var/lib/libvirt/images"
@@ -44,15 +46,6 @@ else:
debug = False
verbose = False
-import importlib
-def dynamic_source_loader(name):
- name = name[0].upper() + name[1:]
- modname = "libvirt_sandbox.image.sources." + name + "Source"
- mod = importlib.import_module(modname)
- classname = name + "Source"
- classimpl = getattr(mod, classname)
- return classimpl()
-
gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale")
gettext.textdomain("libvirt-sandbox")
try:
@@ -73,11 +66,10 @@ def info(msg):
def download(args):
try:
- dynamic_source_loader(args.source).download_template(templatename=args.template,
- templatedir=args.template_dir,
- registry=args.registry,
- username=args.username,
- password=args.password)
+ tmpl = template.Template.from_uri(args.template)
+ source = tmpl.get_source_impl()
+ source.download_template(template=tmpl,
+ templatedir=args.template_dir)
except IOError,e:
print "Source %s cannot be found in given path" %args.source
except Exception,e:
@@ -85,17 +77,21 @@ def download(args):
def delete(args):
try:
- dynamic_source_loader(args.source).delete_template(templatename=args.template,
- templatedir=args.template_dir)
+ tmpl = template.Template.from_uri(args.template)
+ source = tmpl.get_source_impl()
+ source.delete_template(template=tmpl,
+ templatedir=args.template_dir)
except Exception,e:
print "Delete Error %s", str(e)
def create(args):
try:
- dynamic_source_loader(args.source).create_template(templatename=args.template,
- templatedir=args.template_dir,
- connect=args.connect,
- format=args.format)
+ tmpl = template.Template.from_uri(args.template)
+ source = tmpl.get_source_impl()
+ source.create_template(template=tmpl,
+ templatedir=args.template_dir,
+ connect=args.connect,
+ format=args.format)
except Exception,e:
print "Create Error %s" % str(e)
@@ -103,19 +99,22 @@ def run(args):
try:
if args.connect is not None:
check_connect(args.connect)
- source = dynamic_source_loader(args.source)
+
+ tmpl = template.Template.from_uri(args.template)
+ source = tmpl.get_source_impl()
+
name = args.name
if name is None:
randomid = ''.join(random.choice(string.lowercase) for i in range(10))
- name = args.template + ":" + randomid
+ name = tmpl.path[1:] + ":" + randomid
- diskfile = source.get_disk(templatename=args.template,
+ diskfile = source.get_disk(template=tmpl,
templatedir=args.template_dir,
imagedir=args.image_dir,
sandboxname=name)
format = "qcow2"
- commandToRun = source.get_command(args.template, args.template_dir, args.args)
+ commandToRun = source.get_command(tmpl, args.template_dir, args.args)
if len(commandToRun) == 0:
commandToRun = ["/bin/sh"]
cmd = ['virt-sandbox', '--name', name]
@@ -129,7 +128,7 @@ def run(args):
params.append('-N')
params.append(networkArgs)
- allEnvs = source.get_env(args.template, args.template_dir)
+ allEnvs = source.get_env(tmpl, args.template_dir)
envArgs = args.env
if envArgs is not None:
allEnvs = allEnvs + envArgs
@@ -151,7 +150,7 @@ def run(args):
def requires_template(parser):
parser.add_argument("template",
- help=_("name of the template"))
+ help=_("URI of the template"))
def requires_name(parser):
parser.add_argument("-n","--name",
@@ -163,23 +162,10 @@ def check_connect(connectstr):
raise ValueError("URI '%s' is not supported by virt-sandbox-image" % connectstr)
return True
-def requires_source(parser):
- parser.add_argument("-s","--source",
- default="docker",
- help=_("name of the template"))
-
def requires_connect(parser):
parser.add_argument("-c","--connect",
help=_("Connect string for libvirt"))
-def requires_auth_conn(parser):
- parser.add_argument("-r","--registry",
- help=_("Url of the custom registry"))
- parser.add_argument("-u","--username",
- help=_("Username for the custom registry"))
- parser.add_argument("-p","--password",
- help=_("Password for the custom registry"))
-
def requires_template_dir(parser):
global default_template_dir
parser.add_argument("-t","--template-dir",
@@ -196,8 +182,6 @@ def gen_download_args(subparser):
parser = subparser.add_parser("download",
help=_("Download template data"))
requires_template(parser)
- requires_source(parser)
- requires_auth_conn(parser)
requires_template_dir(parser)
parser.set_defaults(func=download)
@@ -205,7 +189,6 @@ def gen_delete_args(subparser):
parser = subparser.add_parser("delete",
help=_("Delete template data"))
requires_template(parser)
- requires_source(parser)
requires_template_dir(parser)
parser.set_defaults(func=delete)
@@ -213,7 +196,6 @@ def gen_create_args(subparser):
parser = subparser.add_parser("create",
help=_("Create image from template data"))
requires_template(parser)
- requires_source(parser)
requires_connect(parser)
requires_template_dir(parser)
parser.add_argument("-f","--format",
@@ -226,7 +208,6 @@ def gen_run_args(subparser):
help=_("Run an already built image"))
requires_name(parser)
requires_template(parser)
- requires_source(parser)
requires_connect(parser)
requires_template_dir(parser)
requires_image_dir(parser)
diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py
index c374a0c..10f8537 100644
--- a/libvirt-sandbox/image/sources/DockerSource.py
+++ b/libvirt-sandbox/image/sources/DockerSource.py
@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015 Universitat Politècnica de Catalunya.
+# Copyright (C) 2015 Red Hat, Inc
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -28,6 +29,8 @@ import traceback
import os
import subprocess
import shutil
+import urlparse
+
class DockerConfParser():
@@ -47,12 +50,6 @@ class DockerConfParser():
class DockerSource(Source):
- www_auth_username = None
- www_auth_password = None
-
- def __init__(self):
- self.default_index_server = "index.docker.io"
-
def _check_cert_validate(self):
major = sys.version_info.major
SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher"
@@ -62,43 +59,38 @@ class DockerSource(Source):
if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion):
sys.stderr.write(SSL_WARNING)
- def download_template(self, templatename, templatedir,
- registry=None, username=None, password=None):
- if registry is None:
- registry = self.default_index_server
-
- if username is not None:
- self.www_auth_username = username
- self.www_auth_password = password
-
+ def download_template(self, template, templatedir):
self._check_cert_validate()
- tag = "latest"
- offset = templatename.find(':')
- if offset != -1:
- tag = templatename[offset + 1:]
- templatename = templatename[0:offset]
+
try:
- (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images",
- {"X-Docker-Token": "true"})
+ (data, res) = self._get_json(template,
+ None,
+ "/v1/repositories" + template.path + "/images",
+ {"X-Docker-Token": "true"})
except urllib2.HTTPError, e:
- raise ValueError(["Image '%s' does not exist" % templatename])
+ raise ValueError(["Image '%s' does not exist" % template])
registryendpoint = res.info().getheader('X-Docker-Endpoints')
token = res.info().getheader('X-Docker-Token')
checksums = {}
for layer in data:
pass
- (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags",
- { "Authorization": "Token " + token })
+ (data, res) = self._get_json(template,
+ registryendpoint,
+ "/v1/repositories" + template.path + "/tags",
+ { "Authorization": "Token " + token })
cookie = res.info().getheader('Set-Cookie')
+ tag = template.params.get("tag", "latest")
if not tag in data:
- raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)])
+ raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, template)])
imagetagid = data[tag]
- (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry",
- { "Authorization": "Token "+token })
+ (data, res) = self._get_json(template,
+ registryendpoint,
+ "/v1/images/" + imagetagid + "/ancestry",
+ { "Authorization": "Token "+ token })
if data[0] != imagetagid:
raise ValueError(["Expected first layer id '%s' to match image id '%s'",
@@ -118,8 +110,11 @@ class DockerSource(Source):
datafile = layerdir + "/template.tar.gz"
if not os.path.exists(jsonfile) or not os.path.exists(datafile):
- res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json",
- { "Authorization": "Token " + token }, jsonfile)
+ res = self._save_data(template,
+ registryendpoint,
+ "/v1/images/" + layerid + "/json",
+ { "Authorization": "Token " + token },
+ jsonfile)
createdFiles.append(jsonfile)
layersize = int(res.info().getheader("Content-Length"))
@@ -128,12 +123,15 @@ class DockerSource(Source):
if layerid in checksums:
datacsum = checksums[layerid]
- self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer",
- { "Authorization": "Token "+token }, datafile, datacsum, layersize)
+ self._save_data(template,
+ registryendpoint,
+ "/v1/images/" + layerid + "/layer",
+ { "Authorization": "Token "+token },
+ datafile, datacsum, layersize)
createdFiles.append(datafile)
index = {
- "name": templatename,
+ "name": template.path,
}
indexfile = templatedir + "/" + imagetagid + "/index.json"
@@ -152,9 +150,11 @@ class DockerSource(Source):
shutil.rmtree(d)
except:
pass
- def _save_data(self,server, path, headers, dest, checksum=None, datalen=None):
+
+ def _save_data(self, template, server, path, headers,
+ dest, checksum=None, datalen=None):
try:
- res = self._get_url(server, path, headers)
+ res = self._get_url(template, server, path, headers)
csum = None
if checksum is not None:
@@ -193,8 +193,22 @@ class DockerSource(Source):
debug("FAIL %s\n" % str(e))
raise
- def _get_url(self,server, path, headers):
- url = "https://" + server + path
+ def _get_url(self, template, server, path, headers):
+ if template.protocol is None:
+ protocol = "https"
+ else:
+ protocol = template.protocol
+
+ if server is None:
+ if template.hostname is None:
+ server = "index.docker.io"
+ else:
+ if template.port is not None:
+ server = template.hostname + ":" + template.port
+ else:
+ server = template.hostname
+
+ url = urlparse.urlunparse((protocol, server, path, None, None, None))
debug("Fetching %s..." % url)
req = urllib2.Request(url=url)
@@ -204,16 +218,18 @@ class DockerSource(Source):
req.add_header(h, headers[h])
#www Auth header starts
- if self.www_auth_username is not None:
- base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '')
+ if template.username and template.password:
+ base64string = base64.encodestring(
+ '%s:%s' % (template.username,
+ template.password)).replace('\n', '')
req.add_header("Authorization", "Basic %s" % base64string)
#www Auth header finish
return urllib2.urlopen(req)
- def _get_json(self,server, path, headers):
+ def _get_json(self, template, server, path, headers):
try:
- res = self._get_url(server, path, headers)
+ res = self._get_url(template, server, path, headers)
data = json.loads(res.read())
debug("OK\n")
return (data, res)
@@ -221,11 +237,11 @@ class DockerSource(Source):
debug("FAIL %s\n" % str(e))
raise
- def create_template(self, templatename, templatedir, connect=None, format=None):
+ def create_template(self, template, templatedir, connect=None, format=None):
if format is None:
format = self.default_disk_format
self._check_disk_format(format)
- imagelist = self._get_image_list(templatename,templatedir)
+ imagelist = self._get_image_list(template, templatedir)
imagelist.reverse()
parentImage = None
@@ -252,7 +268,7 @@ class DockerSource(Source):
if not format in supportedFormats:
raise ValueError(["Unsupported image format %s" % format])
- def _get_image_list(self,templatename,destdir):
+ def _get_image_list(self, template, destdir):
imageparent = {}
imagenames = {}
imagedirs = os.listdir(destdir)
@@ -265,13 +281,13 @@ class DockerSource(Source):
jsonfile = destdir + "/" + imagetagid + "/template.json"
if os.path.exists(jsonfile):
with open(jsonfile,"r") as f:
- template = json.load(f)
- parent = template.get("parent",None)
+ data = json.load(f)
+ parent = data.get("parent",None)
if parent:
imageparent[imagetagid] = parent
- if not templatename in imagenames:
- raise ValueError(["Image %s does not exist locally" %templatename])
- imagetagid = imagenames[templatename]
+ if not template.path in imagenames:
+ raise ValueError(["Image %s does not exist locally" % template.path])
+ imagetagid = imagenames[template.path]
imagelist = []
while imagetagid != None:
imagelist.append(imagetagid)
@@ -310,7 +326,7 @@ class DockerSource(Source):
cmd = cmd + params
subprocess.call(cmd)
- def delete_template(self, templatename, templatedir):
+ def delete_template(self, template, templatedir):
imageusage = {}
imageparent = {}
imagenames = {}
@@ -324,9 +340,9 @@ class DockerSource(Source):
jsonfile = templatedir + "/" + imagetagid + "/template.json"
if os.path.exists(jsonfile):
with open(jsonfile,"r") as f:
- template = json.load(f)
+ data = json.load(f)
- parent = template.get("parent",None)
+ parent = data.get("parent",None)
if parent:
if parent not in imageusage:
imageusage[parent] = []
@@ -334,10 +350,10 @@ class DockerSource(Source):
imageparent[imagetagid] = parent
- if not templatename in imagenames:
- raise ValueError(["Image %s does not exist locally" %templatename])
+ if not template.path in imagenames:
+ raise ValueError(["Image %s does not exist locally" % template.path])
- imagetagid = imagenames[templatename]
+ imagetagid = imagenames[template.path]
while imagetagid != None:
debug("Remove %s\n" % imagetagid)
parent = imageparent.get(imagetagid,None)
@@ -360,15 +376,15 @@ class DockerSource(Source):
parent = None
imagetagid = parent
- def _get_template_data(self, templatename, templatedir):
- imageList = self._get_image_list(templatename, templatedir)
+ def _get_template_data(self, template, templatedir):
+ imageList = self._get_image_list(template, templatedir)
toplayer = imageList[0]
diskfile = templatedir + "/" + toplayer + "/template.qcow2"
configfile = templatedir + "/" + toplayer + "/template.json"
return configfile, diskfile
- def get_disk(self,templatename, templatedir, imagedir, sandboxname):
- configfile, diskfile = self._get_template_data(templatename, templatedir)
+ def get_disk(self, template, templatedir, imagedir, sandboxname):
+ configfile, diskfile = self._get_template_data(template, templatedir)
tempfile = imagedir + "/" + sandboxname + ".qcow2"
if not os.path.exists(imagedir):
os.makedirs(imagedir)
@@ -379,8 +395,8 @@ class DockerSource(Source):
subprocess.call(cmd)
return tempfile
- def get_command(self, templatename, templatedir, userargs):
- configfile, diskfile = self._get_template_data(templatename, templatedir)
+ def get_command(self, template, templatedir, userargs):
+ configfile, diskfile = self._get_template_data(template, templatedir)
configParser = DockerConfParser(configfile)
cmd = configParser.getCommand()
entrypoint = configParser.getEntrypoint()
@@ -393,8 +409,8 @@ class DockerSource(Source):
else:
return entrypoint + cmd
- def get_env(self, templatename, templatedir):
- configfile, diskfile = self._get_template_data(templatename, templatedir)
+ def get_env(self, template, templatedir):
+ configfile, diskfile = self._get_template_data(template, templatedir)
configParser = DockerConfParser(configfile)
return configParser.getEnvs()
diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py
index 8a21f90..597a7fb 100644
--- a/libvirt-sandbox/image/sources/Source.py
+++ b/libvirt-sandbox/image/sources/Source.py
@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2015 Universitat Politècnica de Catalunya.
+# Copyright (C) 2015 Red Hat, Inc
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -33,14 +34,10 @@ class Source():
pass
@abstractmethod
- def download_template(self, templatename, templatedir,
- registry=None, username=None, password=None):
+ def download_template(self, template, templatedir):
"""
- :param templatename: name of the template image to download
+ :param template: libvirt_sandbox.template.Template object
:param templatedir: local directory path in which to store the template
- :param registry: optional hostname of image registry server
- :param username: optional username to authenticate against registry server
- :param password: optional password to authenticate against registry server
Download a template from the registry, storing it in the local
filesystem
@@ -48,10 +45,10 @@ class Source():
pass
@abstractmethod
- def create_template(self, templatename, templatedir,
+ def create_template(self, template, templatedir,
connect=None, format=None):
"""
- :param templatename: name of the template image to create
+ :param template: libvirt_sandbox.template.Template object
:param templatedir: local directory path in which to store the template
:param connect: libvirt connection URI
:param format: disk image format
@@ -63,9 +60,9 @@ class Source():
pass
@abstractmethod
- def delete_template(self, templatename, templatedir):
+ def delete_template(self, template, templatedir):
"""
- :param templatename: name of the template image to delete
+ :param template: libvirt_sandbox.template.Template object
:param templatedir: local directory path from which to delete template
Delete all local files associated with the template
@@ -73,9 +70,9 @@ class Source():
pass
@abstractmethod
- def get_command(self, templatename, templatedir, userargs):
+ def get_command(self, template, templatedir, userargs):
"""
- :param templatename: name of the template image to query
+ :param template: libvirt_sandbox.template.Template object
:param templatedir: local directory path in which templates are stored
:param userargs: user specified arguments to run
@@ -85,9 +82,9 @@ class Source():
pass
@abstractmethod
- def get_disk(self,templatename, templatedir, imagedir, sandboxname):
+ def get_disk(self, template, templatedir, imagedir, sandboxname):
"""
- :param templatename: name of the template image to download
+ :param template: libvirt_sandbox.template.Template object
:param templatedir: local directory path in which to find template
:param imagedir: local directory in which to storage disk image
@@ -97,9 +94,9 @@ class Source():
pass
@abstractmethod
- def get_env(self,templatename, templatedir):
+ def get_env(self, template, templatedir):
"""
- :param templatename: name of the template image to download
+ :param template: libvirt_sandbox.template.Template object
:param templatedir: local directory path in which to find template
Get the dict of environment variables to set
diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py
new file mode 100644
index 0000000..0ad767b
--- /dev/null
+++ b/libvirt-sandbox/image/template.py
@@ -0,0 +1,110 @@
+#
+# -*- coding: utf-8 -*-
+# Authors: Daniel P. Berrange <berrange(a)redhat.com>
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+
+import urlparse
+import importlib
+
+class Template(object):
+
+ def __init__(self,
+ source, protocol,
+ hostname, port,
+ username, password,
+ path, params):
+ """
+ :param source: template source name
+ :param protocol: network transport protocol or None
+ :param hostname: registry hostname or None
+ :param port: registry port or None
+ :param username: username or None
+ :param password: password or None
+ :param path: template path identifier
+ :param params: template parameters
+
+ docker:///ubuntu
+
+ docker+https://index.docker.io/ubuntu?tag=latest
+ """
+
+ self.source = source
+ self.protocol = protocol
+ self.hostname = hostname
+ self.port = port
+ self.username = username
+ self.password = password
+ self.path = path
+ self.params = params
+ if self.params is None:
+ self.params = {}
+
+ def get_source_impl(self):
+ mod = importlib.import_module(
+ "libvirt_sandbox.image.sources." +
+ self.source.capitalize() + "Source")
+ classname = self.source.capitalize() + "Source"
+ classimpl = getattr(mod, classname)
+ return classimpl()
+
+ def __repr__(self):
+ if self.protocol is not None:
+ scheme = self.source + "+" + self.protocol
+ else:
+ scheme = self.source
+ if self.hostname:
+ if self.port:
+ netloc = self.hostname + ":" + self.port
+ else:
+ netloc = self.hostname
+
+ if self.username:
+ if self.password:
+ auth = self.username + ":" + self.password
+ else:
+ auth = self.username
+ netloc = auth + "@" + netloc
+ else:
+ netloc = None
+
+ query = "&".join([key + "=" + self.params[key] for key in self.params.keys()])
+ return urlparse.urlunparse((scheme, netloc, self.path, None, query, None))
+
+ @classmethod
+ def from_uri(klass, uri):
+ o = urlparse.urlparse(uri)
+
+ idx = o.scheme.find("+")
+ if idx == -1:
+ source = o.scheme
+ protocol = None
+ else:
+ source = o.scheme[0:idx]
+ protocol = o.schema[idx + 1:]
+
+ query = {}
+ if o.query is not None and o.query != "":
+ for param in o.query.split("&"):
+ (key, val) = param.split("=")
+ query[key] = val
+ return klass(source, protocol,
+ o.hostname, o.port,
+ o.username, o.password,
+ o.path, query)
+
--
2.4.3
