From: Daniel P Berrange <berrange(a)redhat.com&gt; virt-sandbox-image.py is a python script that lets you download Docker images easily. It is a proof of concept code and consumes Docker Rest API. --- po/POTFILES.in | 1 + virt-sandbox-image/virt-sandbox-image.py | 394 +++++++++++++++++++++++++++++++ 2 files changed, 395 insertions(+) create mode 100644 virt-sandbox-image/virt-sandbox-image.py diff --git a/po/POTFILES.in b/po/POTFILES.in index afcb050..7204112 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -11,3 +11,4 @@ libvirt-sandbox/libvirt-sandbox-context-interactive.c libvirt-sandbox/libvirt-sandbox-init-common.c libvirt-sandbox/libvirt-sandbox-rpcpacket.c libvirt-sandbox/libvirt-sandbox-util.c +virt-sandbox-image/virt-sandbox-image.py diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py new file mode 100644 index 0000000..4f5443b --- /dev/null +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -0,0 +1,394 @@ +#!/usr/bin/python -Es +# +# Authors: Daniel P. Berrange <berrange(a)redhat.com&gt; +# +# Copyright (C) 2013 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# + +import argparse +import gettext +import hashlib +import json +import os +import os.path +import shutil +import sys +import urllib2 +import subprocess + +default_index_server = "index.docker.io" +default_template_dir = "/var/lib/libvirt/templates" + +debug = True +verbose = True + +gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") +gettext.textdomain("libvirt-sandbox") +try: + gettext.install("libvirt-sandbox", + localedir="/usr/share/locale", + unicode=False, + codeset = 'utf-8') +except IOError: + import __builtin__ + __builtin__.__dict__['_'] = unicode + + +def debug(msg): + sys.stderr.write(msg) + +def info(msg): + sys.stdout.write(msg) + +def get_url(server, path, headers): + url = "https://" + server + path + debug(" Fetching %s..." % url) + req = urllib2.Request(url=url) + + if json: + req.add_header("Accept", "application/json") + + for h in headers.keys(): + req.add_header(h, headers[h]) + + return urllib2.urlopen(req) + +def get_json(server, path, headers): + try: + res = get_url(server, path, headers) + data = json.loads(res.read()) + debug("OK\n") + return (data, res) + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + +def save_data(server, path, headers, dest, checksum=None, datalen=None): + try: + res = get_url(server, path, headers) + + csum = None + if checksum is not None: + csum = hashlib.sha256() + + pattern = [".", "o", "O", "o"] + patternIndex = 0 + donelen = 0 + + with open(dest, "w") as f: + while 1: + buf = res.read(1024*64) + if not buf: + break + if csum is not None: + csum.update(buf) + f.write(buf) + + if datalen is not None: + donelen = donelen + len(buf) + debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( + pattern[patternIndex], (donelen/1024), (datalen/1024) + )) + patternIndex = (patternIndex + 1) % 4 + + debug("\x1b[K") + if csum is not None: + csumstr = "sha256:" + csum.hexdigest() + if csumstr != checksum: + debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) + os.remove(dest) + raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) + debug("OK\n") + return res + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + + +def download_template(name, server, destdir): + tag = "latest" + + offset = name.find(':') + if offset != -1: + tag = name[offset + 1:] + name = name[0:offset] + + # First we must ask the index server about the image name. THe + # index server will return an auth token we can use when talking + # to the registry server. We need this token even when anonymous + try: + (data, res) = get_json(server, "/v1/repositories/" + name + "/images", + {"X-Docker-Token": "true"}) + except urllib2.HTTPError, e: + raise ValueError(["Image '%s' does not exist" % name]) + + registryserver = res.info().getheader('X-Docker-Endpoints') + token = res.info().getheader('X-Docker-Token') + checksums = {} + for layer in data: + pass + # XXX Checksums here don't appear to match the data in + # image download later. Find out what these are sums of + #checksums[layer["id"]] = layer["checksum"] + + # Now we ask the registry server for the list of tags associated + # with the image. Tags usually reflect some kind of version of + # the image, but they aren't officially "versions". There is + # always a "latest" tag which is the most recent upload + # + # We must pass in the auth token from the index server. This + # token can only be used once, and we're given a cookie back + # in return to use for later RPC calls. + (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags", + { "Authorization": "Token " + token }) + + cookie = res.info().getheader('Set-Cookie') + + if not tag in data: + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)]) + imagetagid = data[tag] + + # Only base images are self-contained, most images reference one + # or more parents, in a linear stack. Here we are getting the list + # of layers for the image with the tag we used. + (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry", + { "Cookie": cookie }) + + if data[0] != imagetagid: + raise ValueError(["Expected first layer id '%s' to match image id '%s'", + data[0], imagetagid]) + + try: + createdFiles = [] + createdDirs = [] + + for layerid in data: + templatedir = destdir + "/" + layerid + if not os.path.exists(templatedir): + os.mkdir(templatedir) + createdDirs.append(templatedir) + + jsonfile = templatedir + "/template.json" + datafile = templatedir + "/template.tar.gz" + + if not os.path.exists(jsonfile) or not os.path.exists(datafile): + # The '/json' URL gives us some metadata about the layer + res = save_data(registryserver, "/v1/images/" + layerid + "/json", + { "Cookie": cookie }, jsonfile) + createdFiles.append(jsonfile) + layersize = int(res.info().getheader("x-docker-size")) + + datacsum = None + if layerid in checksums: + datacsum = checksums[layerid] + + # and the '/layer' URL is the actual payload, provided + # as a tar.gz archive + save_data(registryserver, "/v1/images/" + layerid + "/layer", + { "Cookie": cookie }, datafile, datacsum, layersize) + createdFiles.append(datafile) + + # Strangely the 'json' data for a layer doesn't include + # its actual name, so we save that in a json file of our own + index = { + "name": name, + } + + indexfile = destdir + "/" + imagetagid + "/index.json" + with open(indexfile, "w") as f: + f.write(json.dumps(index)) + except Exception, e: + for f in createdFiles: + try: + os.remove(f) + except: + pass + for d in createdDirs: + try: + os.rmdir(d) + except: + pass + + +def delete_template(name, destdir): + imageusage = {} + imageparent = {} + imagenames = {} + imagedirs = os.listdir(destdir) + for imagetagid in imagedirs: + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile, "r") as f: + index = json.load(f) + imagenames[index["name"]] = imagetagid + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + with open(jsonfile, "r") as f: + template = json.load(f) + + parent = template.get("parent", None) + if parent: + if parent not in imageusage: + imageusage[parent] = [] + imageusage[parent].append(imagetagid) + imageparent[imagetagid] = parent + + if not name in imagenames: + raise ValueError(["Image %s does not exist locally" % name]) + + imagetagid = imagenames[name] + while imagetagid != None: + debug("Remove %s\n" % imagetagid) + parent = imageparent.get(imagetagid, None) + + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + os.remove(indexfile) + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + os.remove(jsonfile) + datafile = destdir + "/" + imagetagid + "/template.tar.gz" + if os.path.exists(datafile): + os.remove(datafile) + imagedir = destdir + "/" + imagetagid + os.rmdir(imagedir) + + if parent: + if len(imageusage[parent]) != 1: + debug("Parent %s is shared\n" % parent) + parent = None + imagetagid = parent + + +def get_image_list(name, destdir): + imageparent = {} + imagenames = {} + imagedirs = os.listdir(destdir) + for imagetagid in imagedirs: + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile, "r") as f: + index = json.load(f) + imagenames[index["name"]] = imagetagid + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + with open(jsonfile, "r") as f: + template = json.load(f) + + parent = template.get("parent", None) + if parent: + imageparent[imagetagid] = parent + + if not name in imagenames: + raise ValueError(["Image %s does not exist locally" % name]) + + imagetagid = imagenames[name] + imagelist = [] + while imagetagid != None: + imagelist.append(imagetagid) + parent = imageparent.get(imagetagid, None) + imagetagid = parent + + return imagelist + +def create_template(name, imagepath, format, destdir): + if not format in ["qcow2"]: + raise ValueError(["Unsupported image format %s" % format]) + + imagelist = get_image_list(name, destdir) + imagelist.reverse() + + parentImage = None + for imagetagid in imagelist: + templateImage = destdir + "/" + imagetagid + "/template." + format + cmd = ["qemu-img", "create", "-f", "qcow2"] + if parentImage is not None: + cmd.append("-o") + cmd.append("backing_fmt=qcow2,backing_file=%s" % parentImage) + cmd.append(templateImage) + if parentImage is None: + cmd.append("10G") + debug("Run %s\n" % " ".join(cmd)) + subprocess.call(cmd) + parentImage = templateImage + +def download(args): + info("Downloading %s from %s to %s\n" % (args.name, default_index_server, default_template_dir)) + download_template(args.name, default_index_server, default_template_dir) + +def delete(args): + info("Deleting %s from %s\n" % (args.name, default_template_dir)) + delete_template(args.name, default_template_dir) + +def create(args): + info("Creating %s from %s in format %s\n" % (args.imagepath, args.name, args.format)) + create_template(args.name, args.imagepath, args.format, default_template_dir) + +def requires_name(parser): + parser.add_argument("name", + help=_("name of the template")) + +def gen_download_args(subparser): + parser = subparser.add_parser("download", + help=_("Download template data")) + requires_name(parser) + parser.set_defaults(func=download) + +def gen_delete_args(subparser): + parser = subparser.add_parser("delete", + help=_("Delete template data")) + requires_name(parser) + parser.set_defaults(func=delete) + +def gen_create_args(subparser): + parser = subparser.add_parser("create", + help=_("Create image from template data")) + requires_name(parser) + parser.add_argument("imagepath", + help=_("path for image")) + parser.add_argument("format", + help=_("format")) + parser.set_defaults(func=create) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Sandbox Container Image Tool') + + subparser = parser.add_subparsers(help=_("commands")) + gen_download_args(subparser) + gen_delete_args(subparser) + gen_create_args(subparser) + + try: + args = parser.parse_args() + args.func(args) + sys.exit(0) + except KeyboardInterrupt, e: + sys.exit(0) + except ValueError, e: + for line in e: + for l in line: + sys.stderr.write("%s: %s\n" % (sys.argv[0], l)) + sys.stderr.flush() + sys.exit(1) + except IOError, e: + sys.stderr.write("%s: %s: %s\n" % (sys.argv[0], e.filename, e.reason)) + sys.stderr.flush() + sys.exit(1) + except OSError, e: + sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) + sys.stderr.flush() + sys.exit(1) -- 2.1.0

Conflicts: virt-sandbox-image/virt-sandbox-image.py --- virt-sandbox-image/virt-sandbox-image.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index fa9e1c8..55aea6a 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -32,6 +32,14 @@ import sys import urllib2 import subprocess +default_privileged_template_dir = "/var/lib/libvirt/templates" +default_home_dir = os.environ['HOME'] +default_unprivileged_template_dir = default_home_dir + "/.local/share/libvirt/templates" +default_privileged_storage_dir = default_privileged_template_dir + "/storage" +default_unprivileged_storage_dir = default_unprivileged_template_dir + "/storage" +debug = False +verbose = False + import importlib def dynamic_source_loader(name): name = name[0].upper() + name[1:] -- 2.1.0

--- virt-sandbox-image/virt-sandbox-image.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 55aea6a..9e98bf2 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -40,6 +40,8 @@ default_unprivileged_storage_dir = default_unprivileged_template_dir + "/storage debug = False verbose = False +sys.dont_write_byte_code = True + import importlib def dynamic_source_loader(name): name = name[0].upper() + name[1:] -- 2.1.0

These helper functions are for selecting right directories according to running user privileges --- virt-sandbox-image/virt-sandbox-image.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 9e98bf2..5917dd6 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -32,6 +32,8 @@ import sys import urllib2 import subprocess +template_dir = None +storage_dir = None default_privileged_template_dir = "/var/lib/libvirt/templates" default_home_dir = os.environ['HOME'] default_unprivileged_template_dir = default_home_dir + "/.local/share/libvirt/templates" @@ -40,6 +42,29 @@ default_unprivileged_storage_dir = default_unprivileged_template_dir + "/storage debug = False verbose = False +def check_dir_writable(path): + if not os.access(path,os.W_OK): + return False + return True + +def runtime_dir_resolver(): + global default_privileged_template_dir + global default_privileged_storage_dir + global default_unprivileged_template_dir + global default_unprivileged_storage_dir + global template_dir + global storage_dir + if(check_dir_writable(default_privileged_template_dir)): + template_dir = default_privileged_template_dir + storage_dir = default_privileged_storage_dir + return + template_dir = default_unprivileged_template_dir + storage_dir = default_unprivileged_storage_dir + if not os.path.exists(template_dir): + os.makedirs(template_dir) + if not os.path.exists(storage_dir): + os.makedirs(storage_dir) + sys.dont_write_byte_code = True import importlib @@ -380,8 +405,8 @@ def gen_create_args(subparser): parser.set_defaults(func=create) if __name__ == '__main__': + runtime_dir_resolver() parser = argparse.ArgumentParser(description='Sandbox Container Image Tool') - subparser = parser.add_subparsers(help=_("commands")) gen_download_args(subparser) gen_delete_args(subparser) -- 2.1.0

Refactor download function from virt-sandbox-image to use the newly introduced Source abstract class. The docker-specific download code is moved to a new DockerSource class. --- virt-sandbox-image/Makefile.am | 1 + virt-sandbox-image/sources/DockerSource.py | 214 +++++++++++++++++++++++++++++ virt-sandbox-image/sources/Source.py | 4 + virt-sandbox-image/virt-sandbox-image.py | 204 +++++---------------------- 4 files changed, 251 insertions(+), 172 deletions(-) create mode 100644 virt-sandbox-image/sources/DockerSource.py diff --git a/virt-sandbox-image/Makefile.am b/virt-sandbox-image/Makefile.am index 5ab4d2e..8188c80 100644 --- a/virt-sandbox-image/Makefile.am +++ b/virt-sandbox-image/Makefile.am @@ -8,6 +8,7 @@ install-data-local: $(INSTALL) -m 0755 $(srcdir)/virt-sandbox-image.py $(DESTDIR)$(pkgpythondir) $(INSTALL) -m 0644 $(srcdir)/sources/__init__.py $(DESTDIR)$(pkgpythondir)/sources $(INSTALL) -m 0644 $(srcdir)/sources/Source.py $(DESTDIR)$(pkgpythondir)/sources + $(INSTALL) -m 0644 $(srcdir)/sources/DockerSource.py $(DESTDIR)$(pkgpythondir)/sources uninstall-local: rm -f $(DESTDIR)$(pkgpythondir) diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py new file mode 100644 index 0000000..f3cf5f3 --- /dev/null +++ b/virt-sandbox-image/sources/DockerSource.py @@ -0,0 +1,214 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015 Universitat Polit��cnica de Catalunya. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Author: Eren Yagdiran <erenyagdiran(a)gmail.com&gt; +# + +from Source import Source +import urllib2 +import sys +import json +import traceback +import os +import subprocess +import shutil + +class DockerSource(Source): + + www_auth_username = None + www_auth_password = None + + def __init__(self): + self.default_index_server = "index.docker.io" + + def _check_cert_validate(self): + major = sys.version_info.major + SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" + SSL_WARNING +="\nSee https://bugs.python.org/issue22417\n" + py2_7_9_hexversion = 34015728 + py3_4_3_hexversion = 50594800 + if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion): + sys.stderr.write(SSL_WARNING) + + def download_template(self,**args): + name = args['name'] + registry = args['registry'] if args['registry'] is not None else self.default_index_server + username = args['username'] + password = args['password'] + templatedir = args['templatedir'] + self._download_template(name,registry,username,password,templatedir) + + def _download_template(self,name, server,username,password,destdir): + + if username is not None: + self.www_auth_username = username + self.www_auth_password = password + + self._check_cert_validate() + tag = "latest" + offset = name.find(':') + if offset != -1: + tag = name[offset + 1:] + name = name[0:offset] + try: + (data, res) = self._get_json(server, "/v1/repositories/" + name + "/images", + {"X-Docker-Token": "true"}) + except urllib2.HTTPError, e: + raise ValueError(["Image '%s' does not exist" % name]) + + registryserver = res.info().getheader('X-Docker-Endpoints') + token = res.info().getheader('X-Docker-Token') + checksums = {} + for layer in data: + pass + (data, res) = self._get_json(registryserver, "/v1/repositories/" + name + "/tags", + { "Authorization": "Token " + token }) + + cookie = res.info().getheader('Set-Cookie') + + if not tag in data: + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)]) + imagetagid = data[tag] + + (data, res) = self._get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry", + { "Authorization": "Token "+token }) + + if data[0] != imagetagid: + raise ValueError(["Expected first layer id '%s' to match image id '%s'", + data[0], imagetagid]) + + try: + createdFiles = [] + createdDirs = [] + + for layerid in data: + templatedir = destdir + "/" + layerid + if not os.path.exists(templatedir): + os.mkdir(templatedir) + createdDirs.append(templatedir) + + jsonfile = templatedir + "/template.json" + datafile = templatedir + "/template.tar.gz" + + if not os.path.exists(jsonfile) or not os.path.exists(datafile): + res = self._save_data(registryserver, "/v1/images/" + layerid + "/json", + { "Authorization": "Token " + token }, jsonfile) + createdFiles.append(jsonfile) + + layersize = int(res.info().getheader("Content-Length")) + + datacsum = None + if layerid in checksums: + datacsum = checksums[layerid] + + self._save_data(registryserver, "/v1/images/" + layerid + "/layer", + { "Authorization": "Token "+token }, datafile, datacsum, layersize) + createdFiles.append(datafile) + + index = { + "name": name, + } + + indexfile = destdir + "/" + imagetagid + "/index.json" + print("Index file " + indexfile) + with open(indexfile, "w") as f: + f.write(json.dumps(index)) + except Exception as e: + traceback.print_exc() + for f in createdFiles: + try: + os.remove(f) + except: + pass + for d in createdDirs: + try: + shutil.rmtree(d) + except: + pass + def _save_data(self,server, path, headers, dest, checksum=None, datalen=None): + try: + res = self._get_url(server, path, headers) + + csum = None + if checksum is not None: + csum = hashlib.sha256() + + pattern = [".", "o", "O", "o"] + patternIndex = 0 + donelen = 0 + + with open(dest, "w") as f: + while 1: + buf = res.read(1024*64) + if not buf: + break + if csum is not None: + csum.update(buf) + f.write(buf) + + if datalen is not None: + donelen = donelen + len(buf) + debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( + pattern[patternIndex], (donelen/1024), (datalen/1024) + )) + patternIndex = (patternIndex + 1) % 4 + + debug("\x1b[K") + if csum is not None: + csumstr = "sha256:" + csum.hexdigest() + if csumstr != checksum: + debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) + os.remove(dest) + raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) + debug("OK\n") + return res + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + + def _get_url(self,server, path, headers): + url = "https://" + server + path + debug("Fetching %s..." % url) + + req = urllib2.Request(url=url) + if json: + req.add_header("Accept", "application/json") + for h in headers.keys(): + req.add_header(h, headers[h]) + + #www Auth header starts + if self.www_auth_username is not None: + base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '') + req.add_header("Authorization", "Basic %s" % base64string) + #www Auth header finish + + return urllib2.urlopen(req) + + def _get_json(self,server, path, headers): + try: + res = self._get_url(server, path, headers) + data = json.loads(res.read()) + debug("OK\n") + return (data, res) + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + +def debug(msg): + sys.stderr.write(msg) diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py index 508ca80..8751689 100644 --- a/virt-sandbox-image/sources/Source.py +++ b/virt-sandbox-image/sources/Source.py @@ -25,3 +25,7 @@ class Source(): __metaclass__ = ABCMeta def __init__(self): pass + + @abstractmethod + def download_template(self,**args): + pass diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 5917dd6..4f371d7 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -94,176 +94,6 @@ def debug(msg): def info(msg): sys.stdout.write(msg) -def get_url(server, path, headers): - url = "https://" + server + path - debug(" Fetching %s..." % url) - req = urllib2.Request(url=url) - - if json: - req.add_header("Accept", "application/json") - - for h in headers.keys(): - req.add_header(h, headers[h]) - - return urllib2.urlopen(req) - -def get_json(server, path, headers): - try: - res = get_url(server, path, headers) - data = json.loads(res.read()) - debug("OK\n") - return (data, res) - except Exception, e: - debug("FAIL %s\n" % str(e)) - raise - -def save_data(server, path, headers, dest, checksum=None, datalen=None): - try: - res = get_url(server, path, headers) - - csum = None - if checksum is not None: - csum = hashlib.sha256() - - pattern = [".", "o", "O", "o"] - patternIndex = 0 - donelen = 0 - - with open(dest, "w") as f: - while 1: - buf = res.read(1024*64) - if not buf: - break - if csum is not None: - csum.update(buf) - f.write(buf) - - if datalen is not None: - donelen = donelen + len(buf) - debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( - pattern[patternIndex], (donelen/1024), (datalen/1024) - )) - patternIndex = (patternIndex + 1) % 4 - - debug("\x1b[K") - if csum is not None: - csumstr = "sha256:" + csum.hexdigest() - if csumstr != checksum: - debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) - os.remove(dest) - raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) - debug("OK\n") - return res - except Exception, e: - debug("FAIL %s\n" % str(e)) - raise - - -def download_template(name, server, destdir): - tag = "latest" - - offset = name.find(':') - if offset != -1: - tag = name[offset + 1:] - name = name[0:offset] - - # First we must ask the index server about the image name. THe - # index server will return an auth token we can use when talking - # to the registry server. We need this token even when anonymous - try: - (data, res) = get_json(server, "/v1/repositories/" + name + "/images", - {"X-Docker-Token": "true"}) - except urllib2.HTTPError, e: - raise ValueError(["Image '%s' does not exist" % name]) - - registryserver = res.info().getheader('X-Docker-Endpoints') - token = res.info().getheader('X-Docker-Token') - checksums = {} - for layer in data: - pass - # XXX Checksums here don't appear to match the data in - # image download later. Find out what these are sums of - #checksums[layer["id"]] = layer["checksum"] - - # Now we ask the registry server for the list of tags associated - # with the image. Tags usually reflect some kind of version of - # the image, but they aren't officially "versions". There is - # always a "latest" tag which is the most recent upload - # - # We must pass in the auth token from the index server. This - # token can only be used once, and we're given a cookie back - # in return to use for later RPC calls. - (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags", - { "Authorization": "Token " + token }) - - cookie = res.info().getheader('Set-Cookie') - - if not tag in data: - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)]) - imagetagid = data[tag] - - # Only base images are self-contained, most images reference one - # or more parents, in a linear stack. Here we are getting the list - # of layers for the image with the tag we used. - (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry", - { "Authorization": "Token " + token }) - - if data[0] != imagetagid: - raise ValueError(["Expected first layer id '%s' to match image id '%s'", - data[0], imagetagid]) - - try: - createdFiles = [] - createdDirs = [] - - for layerid in data: - templatedir = destdir + "/" + layerid - if not os.path.exists(templatedir): - os.mkdir(templatedir) - createdDirs.append(templatedir) - - jsonfile = templatedir + "/template.json" - datafile = templatedir + "/template.tar.gz" - - if not os.path.exists(jsonfile) or not os.path.exists(datafile): - # The '/json' URL gives us some metadata about the layer - res = save_data(registryserver, "/v1/images/" + layerid + "/json", - { "Authorization": "Token " + token }, jsonfile) - createdFiles.append(jsonfile) - layersize = int(res.info().getheader("Content-Length")) - - datacsum = None - if layerid in checksums: - datacsum = checksums[layerid] - - # and the '/layer' URL is the actual payload, provided - # as a tar.gz archive - save_data(registryserver, "/v1/images/" + layerid + "/layer", - { "Authorization": "Token " + token }, datafile, datacsum, layersize) - createdFiles.append(datafile) - - # Strangely the 'json' data for a layer doesn't include - # its actual name, so we save that in a json file of our own - index = { - "name": name, - } - - indexfile = destdir + "/" + imagetagid + "/index.json" - with open(indexfile, "w") as f: - f.write(json.dumps(index)) - except Exception, e: - for f in createdFiles: - try: - os.remove(f) - except: - pass - for d in createdDirs: - try: - os.rmdir(d) - except: - pass - - def delete_template(name, destdir): imageusage = {} imageparent = {} @@ -367,8 +197,16 @@ def create_template(name, imagepath, format, destdir): parentImage = templateImage def download(args): - info("Downloading %s from %s to %s\n" % (args.name, default_index_server, default_template_dir)) - download_template(args.name, default_index_server, default_template_dir) + try: + dynamic_source_loader(args.source).download_template(name=args.name, + registry=args.registry, + username=args.username, + password=args.password, + templatedir=args.template_dir) + except IOError,e: + print "Source %s cannot be found in given path" %args.source + except Exception,e: + print "Download Error %s" % str(e) def delete(args): info("Deleting %s from %s\n" % (args.name, default_template_dir)) @@ -382,10 +220,32 @@ def requires_name(parser): parser.add_argument("name", help=_("name of the template")) +def requires_source(parser): + parser.add_argument("-s","--source", + default="docker", + help=_("name of the template")) + +def requires_auth_conn(parser): + parser.add_argument("-r","--registry", + help=_("Url of the custom registry")) + parser.add_argument("-u","--username", + help=_("Username for the custom registry")) + parser.add_argument("-p","--password", + help=_("Password for the custom registry")) + +def requires_template_dir(parser): + global template_dir + parser.add_argument("-t","--template-dir", + default=template_dir, + help=_("Template directory for saving templates")) + def gen_download_args(subparser): parser = subparser.add_parser("download", help=_("Download template data")) + requires_source(parser) requires_name(parser) + requires_auth_conn(parser) + requires_template_dir(parser) parser.set_defaults(func=download) def gen_delete_args(subparser): -- 2.1.0

Move the docker-related code to the DockerSource and use the Source mechanism --- virt-sandbox-image/sources/DockerSource.py | 100 +++++++++++++++++++++++++++++ virt-sandbox-image/sources/Source.py | 4 ++ virt-sandbox-image/virt-sandbox-image.py | 76 +++++----------------- 3 files changed, 121 insertions(+), 59 deletions(-) diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py index f3cf5f3..09eea85 100644 --- a/virt-sandbox-image/sources/DockerSource.py +++ b/virt-sandbox-image/sources/DockerSource.py @@ -210,5 +210,105 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise + def create_template(self,**args): + name = args['name'] + connect = args['connect'] + templatedir = args['templatedir'] + format = args['format'] + format = format if format is not None else self.default_disk_format + + self._create_template(name, + connect, + templatedir, + format) + + def _create_template(self,name,connect,templatedir,format): + self._check_disk_format(format) + imagelist = self._get_image_list(name,templatedir) + imagelist.reverse() + + parentImage = None + for imagetagid in imagelist: + templateImage = templatedir + "/" + imagetagid + "/template." + format + cmd = ["qemu-img","create","-f","qcow2"] + if parentImage is not None: + cmd.append("-o") + cmd.append("backing_fmt=qcow2,backing_file=%s" % parentImage) + cmd.append(templateImage) + if parentImage is None: + cmd.append("10G") + subprocess.call(cmd) + + if parentImage is None: + self._format_disk(templateImage,format,connect) + + self._extract_tarballs(templatedir + "/" + imagetagid + "/template.",format,connect) + parentImage = templateImage + + + def _check_disk_format(self,format): + supportedFormats = ['qcow2'] + if not format in supportedFormats: + raise ValueError(["Unsupported image format %s" % format]) + + def _get_image_list(self,name,destdir): + imageparent = {} + imagenames = {} + imagedirs = os.listdir(destdir) + for imagetagid in imagedirs: + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile,"r") as f: + index = json.load(f) + imagenames[index["name"]] = imagetagid + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + with open(jsonfile,"r") as f: + template = json.load(f) + parent = template.get("parent",None) + if parent: + imageparent[imagetagid] = parent + if not name in imagenames: + raise ValueError(["Image %s does not exist locally" %name]) + imagetagid = imagenames[name] + imagelist = [] + while imagetagid != None: + imagelist.append(imagetagid) + parent = imageparent.get(imagetagid,None) + imagetagid = parent + return imagelist + + def _format_disk(self,disk,format,connect): + cmd = ['virt-sandbox'] + if connect is not None: + cmd.append("-c") + cmd.append(connect) + cmd.append("-p") + params = ['--disk=file:disk_image=%s,format=%s' %(disk,format), + '/sbin/mkfs.ext3', + '/dev/disk/by-tag/disk_image'] + cmd = cmd + params + subprocess.call(cmd) + + def _extract_tarballs(self,directory,format,connect): + tempdir = "/mnt" + tarfile = directory + "tar.gz" + diskfile = directory + "qcow2" + cmd = ['virt-sandbox'] + if connect is not None: + cmd.append("-c") + cmd.append(connect) + cmd.append("-p") + params = ['-m', + 'host-image:/mnt=%s,format=%s' %(diskfile,format), + '--', + '/bin/tar', + 'zxf', + '%s' %tarfile, + '-C', + '/mnt'] + cmd = cmd + params + subprocess.call(cmd) + def debug(msg): sys.stderr.write(msg) diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py index 8751689..6ac08dc 100644 --- a/virt-sandbox-image/sources/Source.py +++ b/virt-sandbox-image/sources/Source.py @@ -29,3 +29,7 @@ class Source(): @abstractmethod def download_template(self,**args): pass + + @abstractmethod + def create_template(self,**args): + pass diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 4f371d7..745401c 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -143,59 +143,6 @@ def delete_template(name, destdir): parent = None imagetagid = parent - -def get_image_list(name, destdir): - imageparent = {} - imagenames = {} - imagedirs = os.listdir(destdir) - for imagetagid in imagedirs: - indexfile = destdir + "/" + imagetagid + "/index.json" - if os.path.exists(indexfile): - with open(indexfile, "r") as f: - index = json.load(f) - imagenames[index["name"]] = imagetagid - jsonfile = destdir + "/" + imagetagid + "/template.json" - if os.path.exists(jsonfile): - with open(jsonfile, "r") as f: - template = json.load(f) - - parent = template.get("parent", None) - if parent: - imageparent[imagetagid] = parent - - if not name in imagenames: - raise ValueError(["Image %s does not exist locally" % name]) - - imagetagid = imagenames[name] - imagelist = [] - while imagetagid != None: - imagelist.append(imagetagid) - parent = imageparent.get(imagetagid, None) - imagetagid = parent - - return imagelist - -def create_template(name, imagepath, format, destdir): - if not format in ["qcow2"]: - raise ValueError(["Unsupported image format %s" % format]) - - imagelist = get_image_list(name, destdir) - imagelist.reverse() - - parentImage = None - for imagetagid in imagelist: - templateImage = destdir + "/" + imagetagid + "/template." + format - cmd = ["qemu-img", "create", "-f", "qcow2"] - if parentImage is not None: - cmd.append("-o") - cmd.append("backing_fmt=qcow2,backing_file=%s" % parentImage) - cmd.append(templateImage) - if parentImage is None: - cmd.append("10G") - debug("Run %s\n" % " ".join(cmd)) - subprocess.call(cmd) - parentImage = templateImage - def download(args): try: dynamic_source_loader(args.source).download_template(name=args.name, @@ -213,8 +160,13 @@ def delete(args): delete_template(args.name, default_template_dir) def create(args): - info("Creating %s from %s in format %s\n" % (args.imagepath, args.name, args.format)) - create_template(args.name, args.imagepath, args.format, default_template_dir) + try: + dynamic_source_loader(args.source).create_template(name=args.name, + connect=args.connect, + templatedir=args.template_dir, + format=args.format) + except Exception,e: + print "Create Error %s" % str(e) def requires_name(parser): parser.add_argument("name", @@ -225,6 +177,10 @@ def requires_source(parser): default="docker", help=_("name of the template")) +def requires_connect(parser): + parser.add_argument("-c","--connect", + help=_("Connect string for libvirt")) + def requires_auth_conn(parser): parser.add_argument("-r","--registry", help=_("Url of the custom registry")) @@ -258,10 +214,12 @@ def gen_create_args(subparser): parser = subparser.add_parser("create", help=_("Create image from template data")) requires_name(parser) - parser.add_argument("imagepath", - help=_("path for image")) - parser.add_argument("format", - help=_("format")) + requires_source(parser) + requires_connect(parser) + requires_template_dir(parser) + parser.add_argument("-f","--format", + default="qcow2", + help=_("format format for image")) parser.set_defaults(func=create) if __name__ == '__main__': -- 2.1.0

Provide a way to know how a template can be started depending on the used source DockerSource will need to parse the topmost config file in order to find the igniter command --- virt-sandbox-image/sources/DockerSource.py | 14 ++++++++++++++ virt-sandbox-image/sources/Source.py | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py index 760ba6c..3e0362b 100644 --- a/virt-sandbox-image/sources/DockerSource.py +++ b/virt-sandbox-image/sources/DockerSource.py @@ -29,6 +29,15 @@ import os import subprocess import shutil +class DockerConfParser(): + + def __init__(self,jsonfile): + with open(jsonfile) as json_file: + self.json_data = json.load(json_file) + def getRunCommand(self): + cmd = self.json_data['container_config']['Cmd'][2] + return cmd[cmd.index('"') + 1:cmd.rindex('"')] + class DockerSource(Source): www_auth_username = None @@ -363,5 +372,10 @@ class DockerSource(Source): parent = None imagetagid = parent + def get_command(self,configfile): + configParser = DockerConfParser(configfile) + commandToRun = configParser.getRunCommand() + return commandToRun + def debug(msg): sys.stderr.write(msg) diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py index d66e61e..9daf62d 100644 --- a/virt-sandbox-image/sources/Source.py +++ b/virt-sandbox-image/sources/Source.py @@ -37,3 +37,7 @@ class Source(): @abstractmethod def delete_template(self,**args): pass + + @abstractmethod + def get_command(self,**args): + pass -- 2.1.0

On Tue, 2015-09-08 at 17:21 +0100, Daniel P. Berrange wrote: Do we really want to have a default to /bin/sh when docker is giving an error to the user? Error response from daemon: No command specified -- Cedric

Commandline parameters for running a template --- virt-sandbox-image/virt-sandbox-image.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 1da5150..d6b682f 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -178,6 +178,18 @@ def gen_create_args(subparser): help=_("format format for image")) parser.set_defaults(func=create) +def gen_run_args(subparser): + parser = subparser.add_parser("run", + help=_("Run a already built image")) + requires_name(parser) + requires_source(parser) + requires_connect(parser) + parser.add_argument("-t","--template-dir", + help=_("Template directory for saving templates")) + parser.add_argument("-i","--igniter", + help=_("Igniter command for image")) + parser.set_defaults(func=run) + if __name__ == '__main__': runtime_dir_resolver() parser = argparse.ArgumentParser(description='Sandbox Container Image Tool') @@ -185,6 +197,7 @@ if __name__ == '__main__': gen_download_args(subparser) gen_delete_args(subparser) gen_create_args(subparser) + gen_run_args(subparser) try: args = parser.parse_args() -- 2.1.0

On Fri, Aug 28, 2015 at 01:47:39PM +0000, Eren Yagdiran wrote: Rather than having an --igniter arg, which only allows a single binary name, we should just allow an arbitrary number of positional args, so the user can pass full arg lists, eg virt-sandbox-image run ubuntu /bin/somefile somearg otherarg .... Regards, Daniel -- |: http://berrange.com -o- http://www.flickr.com/photos/dberrange/ :| |: http://libvirt.org -o- http://virt-manager.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: http://entangle-photo.org -o- http://live.gnome.org/gtk-vnc :|

Provide a way to know which disk image to use for the sandbox depending on the used source DockerSource will need to locate the topmost disk image among all the layers images --- virt-sandbox-image/sources/DockerSource.py | 16 ++++++++++++++++ virt-sandbox-image/sources/Source.py | 4 ++++ virt-sandbox-image/virt-sandbox-image.py | 9 +++++++++ 3 files changed, 29 insertions(+) diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py index 3e0362b..87fbcf3 100644 --- a/virt-sandbox-image/sources/DockerSource.py +++ b/virt-sandbox-image/sources/DockerSource.py @@ -372,6 +372,22 @@ class DockerSource(Source): parent = None imagetagid = parent + def get_disk(self,**args): + name = args['name'] + destdir = args['templatedir'] + sandboxid = args['id'] + imageList = self._get_image_list(name,destdir) + toplayer = imageList[0] + diskfile = destdir + "/" + toplayer + "/template.qcow2" + configfile = destdir + "/" + toplayer + "/template.json" + tempfile = destdir + "/" + toplayer + "/" + sandboxid + ".qcow2" + cmd = ["qemu-img","create","-q","-f","qcow2"] + cmd.append("-o") + cmd.append("backing_fmt=qcow2,backing_file=%s" % diskfile) + cmd.append(tempfile) + subprocess.call(cmd) + return (tempfile,configfile) + def get_command(self,configfile): configParser = DockerConfParser(configfile) commandToRun = configParser.getRunCommand() diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py index 9daf62d..9a3da59 100644 --- a/virt-sandbox-image/sources/Source.py +++ b/virt-sandbox-image/sources/Source.py @@ -41,3 +41,7 @@ class Source(): @abstractmethod def get_command(self,**args): pass + + @abstractmethod + def get_disk(self,**args): + pass diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index c46abd4..a73619c 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -31,6 +31,8 @@ import shutil import sys import urllib2 import subprocess +import random +import string template_dir = None storage_dir = None @@ -128,6 +130,12 @@ def check_connect(connectstr): raise ValueError("%s is not supported by Virt-sandbox" %connectstr) return True +def requires_id(parser): + randomid = ''.join(random.choice(string.lowercase) for i in range(10)) + parser.add_argument("-d","--id", + default=randomid, + help=_("id of the running sandbox")) + def requires_name(parser): parser.add_argument("name", help=_("name of the template")) @@ -187,6 +195,7 @@ def gen_create_args(subparser): def gen_run_args(subparser): parser = subparser.add_parser("run", help=_("Run a already built image")) + requires_id(parser) requires_name(parser) requires_source(parser) requires_connect(parser) -- 2.1.0

Run an already-built template If there is no execution command specified by user, source.get_command will find the command to invoke --- virt-sandbox-image/virt-sandbox-image.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index a73619c..c182874 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -130,6 +130,31 @@ def check_connect(connectstr): raise ValueError("%s is not supported by Virt-sandbox" %connectstr) return True +def run(args): + try: + if args.connect is not None: + check_connect(args.connect) + source = dynamic_source_loader(args.source) + diskfile,configfile = source.get_disk(name=args.name,templatedir=args.template_dir,id=args.id) + + format = "qcow2" + commandToRun = args.igniter + if commandToRun is None: + commandToRun = source.get_command(configfile) + cmd = ['virt-sandbox'] + if args.connect is not None: + cmd.append("-c") + cmd.append(args.connect) + params = ['-m','host-image:/=%s,format=%s' %(diskfile,format), + '--', + commandToRun] + cmd = cmd + params + subprocess.call(cmd) + subprocess.call(["rm", "-rf", diskfile]) + + except Exception,e: + print "Run Error %s" % str(e) + def requires_id(parser): randomid = ''.join(random.choice(string.lowercase) for i in range(10)) parser.add_argument("-d","--id", -- 2.1.0

Virt-sandbox-image will pass exact network arguments to virt-sandbox --- virt-sandbox-image/virt-sandbox-image.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index c182874..058738a 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -145,9 +145,13 @@ def run(args): if args.connect is not None: cmd.append("-c") cmd.append(args.connect) - params = ['-m','host-image:/=%s,format=%s' %(diskfile,format), - '--', - commandToRun] + params = ['-m','host-image:/=%s,format=%s' %(diskfile,format)] + networkArgs = args.network + if networkArgs is not None: + params.append('-N') + params.append(networkArgs) + params.append('--') + params.append(commandToRun) cmd = cmd + params subprocess.call(cmd) subprocess.call(["rm", "-rf", diskfile]) @@ -228,6 +232,8 @@ def gen_run_args(subparser): help=_("Template directory for saving templates")) parser.add_argument("-i","--igniter", help=_("Igniter command for image")) + parser.add_argument("-n","--network", + help=_("Network params for running template")) parser.set_defaults(func=run) if __name__ == '__main__': -- 2.1.0

Volumes let user to map host-paths into sandbox. Docker containers need volumes for data persistence. --- virt-sandbox-image/sources/DockerSource.py | 12 ++++++++++++ virt-sandbox-image/sources/Source.py | 4 ++++ virt-sandbox-image/virt-sandbox-image.py | 22 ++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py index 87fbcf3..1022107 100644 --- a/virt-sandbox-image/sources/DockerSource.py +++ b/virt-sandbox-image/sources/DockerSource.py @@ -28,6 +28,7 @@ import traceback import os import subprocess import shutil +import collections class DockerConfParser(): @@ -37,6 +38,13 @@ class DockerConfParser(): def getRunCommand(self): cmd = self.json_data['container_config']['Cmd'][2] return cmd[cmd.index('"') + 1:cmd.rindex('"')] + def getVolumes(self): + volumes = self.json_data['container_config']['Volumes'] + volumelist = [] + if isinstance(volumes,collections.Iterable): + for key,value in volumes.iteritems(): + volumelist.append(key) + return volumelist class DockerSource(Source): @@ -393,5 +401,9 @@ class DockerSource(Source): commandToRun = configParser.getRunCommand() return commandToRun + def get_volume(self,configfile): + configParser = DockerConfParser(configfile) + return configParser.getVolumes() + def debug(msg): sys.stderr.write(msg) diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py index 9a3da59..8cc508e 100644 --- a/virt-sandbox-image/sources/Source.py +++ b/virt-sandbox-image/sources/Source.py @@ -45,3 +45,7 @@ class Source(): @abstractmethod def get_disk(self,**args): pass + + @abstractmethod + def get_volume(self,**args): + pass diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 058738a..79f8d8c 100755 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -132,6 +132,7 @@ def check_connect(connectstr): def run(args): try: + global storage_dir if args.connect is not None: check_connect(args.connect) source = dynamic_source_loader(args.source) @@ -150,6 +151,25 @@ def run(args): if networkArgs is not None: params.append('-N') params.append(networkArgs) + allVolumes = source.get_volume(configfile) + volumeArgs = args.volume + if volumeArgs is not None: + allVolumes = allVolumes + volumeArgs + for volume in allVolumes: + volumeSplit = volume.split(":") + volumelen = len(volumeSplit) + if volumelen == 2: + hostPath = volumeSplit[0] + guestPath = volumeSplit[1] + elif volumelen == 1: + guestPath = volumeSplit[0] + hostPath = storage_dir + guestPath + if not os.path.exists(hostPath): + os.makedirs(hostPath) + else: + pass + params.append("--mount") + params.append("host-bind:%s=%s" %(guestPath,hostPath)) params.append('--') params.append(commandToRun) cmd = cmd + params @@ -234,6 +254,8 @@ def gen_run_args(subparser): help=_("Igniter command for image")) parser.add_argument("-n","--network", help=_("Network params for running template")) + parser.add_argument("-v","--volume",action="append", + help=_("Volume params for running template")) parser.set_defaults(func=run) if __name__ == '__main__': -- 2.1.0

On Tue, 2015-09-08 at 17:27 +0100, Daniel P. Berrange wrote: See here for the full syntax / documentation of volume mounts: http://docs.docker.com/userguide/dockervolumes/#volume It seems this commit needs quite some rework: the Config/Volumes section only gets the destination of the bind mounts created by docker, while HostConfig/Mounts gets all the details: "Mounts": [ { "Source": "/public", "Destination": "/home/public", "Mode": "", "RW": true }, { "Source": "/home", "Destination": "ro", "Mode": "", "RW": true }, { "Name": "43c4b4472e74bc5d1cf0a390f8707decade340df5e10cdea468bb191cac14a76", "Source": "/var/lib/docker/volumes/43c4b4472e74bc5d1cf0a390f8707decade340df5e10cdea468bb191cac14a76/_data", "Destination": "/srv/www", "Driver": "local", "Mode": "", "RW": true } ], Note in this sample, that at least some versions of docker can't properly handle the -v /path:ro flag (documented) and try to create a bind mount to /ro ;) In the same config, I'm having this for the Volumes: "Volumes": { "/srv/www": {} }, It really seems like volumes are just bind mounts to an automatically created image. The purpose of this feature is easy data storing outside the rootfs image. Volumes and Mounts are both created with the docker run -v option. The --volumes-from will just add the required items in the Mounts section of the docker container config. Indeed there is no ':' in the Volumes list, and there will be none in the mounts section too. I think Eren mixed up the docker parameters and the config file values. -- Cedric

On Wed, 2015-09-09 at 09:54 +0100, Daniel P. Berrange wrote: Yes, looks like docker build files VOLUME directory only can handle the automatically created data volume: http://docs.docker.com/reference/builder/#volume Thus may be better to ignore bind mounts. Indeed. -- Cedric

On Fri, Aug 28, 2015 at 01:47:46PM +0000, Eren Yagdiran wrote: This should be added to a new version block 0.6.1, rather than changing the existing version. Regards, Daniel -- |: http://berrange.com -o- http://www.flickr.com/photos/dberrange/ :| |: http://libvirt.org -o- http://virt-manager.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: http://entangle-photo.org -o- http://live.gnome.org/gtk-vnc :|

Common-init reads config file and exports custom environment variables from config file and applies them to the running sandbox. --- libvirt-sandbox/libvirt-sandbox-init-common.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libvirt-sandbox/libvirt-sandbox-init-common.c b/libvirt-sandbox/libvirt-sandbox-init-common.c index d35f760..16abf25 100644 --- a/libvirt-sandbox/libvirt-sandbox-init-common.c +++ b/libvirt-sandbox/libvirt-sandbox-init-common.c @@ -336,6 +336,21 @@ static gboolean setup_network(GVirSandboxConfig *config, GError **error) } +static gboolean setup_custom_env(GVirSandboxConfig *config, GError **error) +{ + gboolean ret = FALSE; + GHashTableIter iter; + gpointer key, value; + g_hash_table_iter_init (&iter, gvir_sandbox_config_get_envs(config)); + while (g_hash_table_iter_next (&iter, &key, &value)){ + if(setenv(key,value,1)!=0) + goto cleanup; + } + ret = TRUE; + cleanup: + return ret; +} + static int change_user(const gchar *user, uid_t uid, gid_t gid, @@ -1262,6 +1277,9 @@ int main(int argc, char **argv) { if (!setup_disk_tags()) exit(EXIT_FAILURE); + if (!setup_custom_env(config, &error)) + goto error; + if (!setup_network(config, &error)) goto error; -- 2.1.0