From: Daniel P Berrange <berrange(a)redhat.com&gt; virt-sandbox-image.py is a python script that lets you download Docker images easily. It is a proof of concept code and consumes Docker Rest API. --- po/POTFILES.in | 1 + virt-sandbox-image/virt-sandbox-image.py | 397 +++++++++++++++++++++++++++++++ 2 files changed, 398 insertions(+) create mode 100644 virt-sandbox-image/virt-sandbox-image.py diff --git a/po/POTFILES.in b/po/POTFILES.in index afcb050..7204112 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -11,3 +11,4 @@ libvirt-sandbox/libvirt-sandbox-context-interactive.c libvirt-sandbox/libvirt-sandbox-init-common.c libvirt-sandbox/libvirt-sandbox-rpcpacket.c libvirt-sandbox/libvirt-sandbox-util.c +virt-sandbox-image/virt-sandbox-image.py diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py new file mode 100644 index 0000000..4da3dde --- /dev/null +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -0,0 +1,397 @@ +#!/usr/bin/python -Es +# +# Authors: Daniel P. Berrange <berrange(a)redhat.com&gt; +# +# Copyright (C) 2013 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# + +import argparse +import gettext +import hashlib +import json +import os +import os.path +import shutil +import sys +import urllib2 +import subprocess + +default_index_server = "index.docker.io" +default_template_dir = "/var/lib/libvirt/templates" + +debug = True +verbose = True + +gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") +gettext.textdomain("libvirt-sandbox") +try: + gettext.install("libvirt-sandbox", + localedir="/usr/share/locale", + unicode=False, + codeset = 'utf-8') +except IOError: + import __builtin__ + __builtin__.__dict__['_'] = unicode + + +def debug(msg): + sys.stderr.write(msg) + +def info(msg): + sys.stdout.write(msg) + +def get_url(server, path, headers): + url = "https://" + server + path + debug(" Fetching %s..." % url) + + req = urllib2.Request(url=url) + + if json: + req.add_header("Accept", "application/json") + + for h in headers.keys(): + req.add_header(h, headers[h]) + + return urllib2.urlopen(req) + +def get_json(server, path, headers): + try: + res = get_url(server, path, headers) + data = json.loads(res.read()) + debug("OK\n") + return (data, res) + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + +def save_data(server, path, headers, dest, checksum=None, datalen=None): + try: + res = get_url(server, path, headers) + + csum = None + if checksum is not None: + csum = hashlib.sha256() + + pattern = [".", "o", "O", "o"] + patternIndex = 0 + donelen = 0 + + with open(dest, "w") as f: + while 1: + buf = res.read(1024*64) + if not buf: + break + if csum is not None: + csum.update(buf) + f.write(buf) + + if datalen is not None: + donelen = donelen + len(buf) + debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( + pattern[patternIndex], (donelen/1024), (datalen/1024) + )) + patternIndex = (patternIndex + 1) % 4 + + debug("\x1b[K") + if csum is not None: + csumstr = "sha256:" + csum.hexdigest() + if csumstr != checksum: + debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) + os.remove(dest) + raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) + + debug("OK\n") + return res + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + + +def download_template(name, server, destdir): + tag = "latest" + + offset = name.find(':') + if offset != -1: + tag = name[offset + 1:] + name = name[0:offset] + + # First we must ask the index server about the image name. THe + # index server will return an auth token we can use when talking + # to the registry server. We need this token even when anonymous + try: + (data, res) = get_json(server, "/v1/repositories/" + name + "/images", + {"X-Docker-Token": "true"}) + except urllib2.HTTPError, e: + raise ValueError(["Image '%s' does not exist" % name]) + + registryserver = res.info().getheader('X-Docker-Endpoints') + token = res.info().getheader('X-Docker-Token') + checksums = {} + for layer in data: + pass + # XXX Checksums here don't appear to match the data in + # image download later. Find out what these are sums of + #checksums[layer["id"]] = layer["checksum"] + + # Now we ask the registry server for the list of tags associated + # with the image. Tags usually reflect some kind of version of + # the image, but they aren't officially "versions". There is + # always a "latest" tag which is the most recent upload + # + # We must pass in the auth token from the index server. This + # token can only be used once, and we're given a cookie back + # in return to use for later RPC calls. + (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags", + { "Authorization": "Token " + token }) + + cookie = res.info().getheader('Set-Cookie') + + if not tag in data: + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)]) + imagetagid = data[tag] + + # Only base images are self-contained, most images reference one + # or more parents, in a linear stack. Here we are getting the list + # of layers for the image with the tag we used. + (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry", + { "Cookie": cookie }) + + if data[0] != imagetagid: + raise ValueError(["Expected first layer id '%s' to match image id '%s'", + data[0], imagetagid]) + + try: + createdFiles = [] + createdDirs = [] + + for layerid in data: + templatedir = destdir + "/" + layerid + if not os.path.exists(templatedir): + os.mkdir(templatedir) + createdDirs.append(templatedir) + + jsonfile = templatedir + "/template.json" + datafile = templatedir + "/template.tar.gz" + + if not os.path.exists(jsonfile) or not os.path.exists(datafile): + # The '/json' URL gives us some metadata about the layer + res = save_data(registryserver, "/v1/images/" + layerid + "/json", + { "Cookie": cookie }, jsonfile) + createdFiles.append(jsonfile) + layersize = int(res.info().getheader("x-docker-size")) + + datacsum = None + if layerid in checksums: + datacsum = checksums[layerid] + + # and the '/layer' URL is the actual payload, provided + # as a tar.gz archive + save_data(registryserver, "/v1/images/" + layerid + "/layer", + { "Cookie": cookie }, datafile, datacsum, layersize) + createdFiles.append(datafile) + + # Strangely the 'json' data for a layer doesn't include + # its actual name, so we save that in a json file of our own + index = { + "name": name, + } + + indexfile = destdir + "/" + imagetagid + "/index.json" + with open(indexfile, "w") as f: + f.write(json.dumps(index)) + except Exception, e: + for f in createdFiles: + try: + os.remove(f) + except: + pass + for d in createdDirs: + try: + os.rmdir(d) + except: + pass + + +def delete_template(name, destdir): + imageusage = {} + imageparent = {} + imagenames = {} + imagedirs = os.listdir(destdir) + for imagetagid in imagedirs: + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile, "r") as f: + index = json.load(f) + imagenames[index["name"]] = imagetagid + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + with open(jsonfile, "r") as f: + template = json.load(f) + + parent = template.get("parent", None) + if parent: + if parent not in imageusage: + imageusage[parent] = [] + imageusage[parent].append(imagetagid) + imageparent[imagetagid] = parent + + if not name in imagenames: + raise ValueError(["Image %s does not exist locally" % name]) + + imagetagid = imagenames[name] + while imagetagid != None: + debug("Remove %s\n" % imagetagid) + parent = imageparent.get(imagetagid, None) + + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + os.remove(indexfile) + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + os.remove(jsonfile) + datafile = destdir + "/" + imagetagid + "/template.tar.gz" + if os.path.exists(datafile): + os.remove(datafile) + imagedir = destdir + "/" + imagetagid + os.rmdir(imagedir) + + if parent: + if len(imageusage[parent]) != 1: + debug("Parent %s is shared\n" % parent) + parent = None + imagetagid = parent + + +def get_image_list(name, destdir): + imageparent = {} + imagenames = {} + imagedirs = os.listdir(destdir) + for imagetagid in imagedirs: + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile, "r") as f: + index = json.load(f) + imagenames[index["name"]] = imagetagid + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + with open(jsonfile, "r") as f: + template = json.load(f) + + parent = template.get("parent", None) + if parent: + imageparent[imagetagid] = parent + + if not name in imagenames: + raise ValueError(["Image %s does not exist locally" % name]) + + imagetagid = imagenames[name] + imagelist = [] + while imagetagid != None: + imagelist.append(imagetagid) + parent = imageparent.get(imagetagid, None) + imagetagid = parent + + return imagelist + +def create_template(name, imagepath, format, destdir): + if not format in ["qcow2"]: + raise ValueError(["Unsupported image format %s" % format]) + + imagelist = get_image_list(name, destdir) + imagelist.reverse() + + parentImage = None + for imagetagid in imagelist: + templateImage = destdir + "/" + imagetagid + "/template." + format + cmd = ["qemu-img", "create", "-f", "qcow2"] + if parentImage is not None: + cmd.append("-o") + cmd.append("backing_fmt=qcow2,backing_file=%s" % parentImage) + cmd.append(templateImage) + if parentImage is None: + cmd.append("10G") + debug("Run %s\n" % " ".join(cmd)) + subprocess.call(cmd) + parentImage = templateImage + + +def download(args): + info("Downloading %s from %s to %s\n" % (args.name, default_index_server, default_template_dir)) + download_template(args.name, default_index_server, default_template_dir) + +def delete(args): + info("Deleting %s from %s\n" % (args.name, default_template_dir)) + delete_template(args.name, default_template_dir) + +def create(args): + info("Creating %s from %s in format %s\n" % (args.imagepath, args.name, args.format)) + create_template(args.name, args.imagepath, args.format, default_template_dir) + +def requires_name(parser): + parser.add_argument("name", + help=_("name of the template")) + +def gen_download_args(subparser): + parser = subparser.add_parser("download", + help=_("Download template data")) + requires_name(parser) + parser.set_defaults(func=download) + +def gen_delete_args(subparser): + parser = subparser.add_parser("delete", + help=_("Delete template data")) + requires_name(parser) + parser.set_defaults(func=delete) + +def gen_create_args(subparser): + parser = subparser.add_parser("create", + help=_("Create image from template data")) + requires_name(parser) + parser.add_argument("imagepath", + help=_("path for image")) + parser.add_argument("format", + help=_("format")) + parser.set_defaults(func=create) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Sandbox Container Image Tool') + + subparser = parser.add_subparsers(help=_("commands")) + gen_download_args(subparser) + gen_delete_args(subparser) + gen_create_args(subparser) + + try: + args = parser.parse_args() + args.func(args) + sys.exit(0) + except KeyboardInterrupt, e: + sys.exit(0) + except ValueError, e: + for line in e: + for l in line: + sys.stderr.write("%s: %s\n" % (sys.argv[0], l)) + sys.stderr.flush() + sys.exit(1) + except IOError, e: + sys.stderr.write("%s: %s: %s\n" % (sys.argv[0], e.filename, e.reason)) + sys.stderr.flush() + sys.exit(1) + except OSError, e: + sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) + sys.stderr.flush() + sys.exit(1) -- 2.1.0

Authentication fix for Docker REST API. --- virt-sandbox-image/virt-sandbox-image.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 4da3dde..324e568 100644 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -1,8 +1,10 @@ #!/usr/bin/python -Es # # Authors: Daniel P. Berrange <berrange(a)redhat.com&gt; +# Eren Yagdiran <erenyagdiran(a)gmail.com&gt; # # Copyright (C) 2013 Red Hat, Inc. +# Copyright (C) 2015 Universitat Polit��cnica de Catalunya. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -168,7 +170,7 @@ def download_template(name, server, destdir): # or more parents, in a linear stack. Here we are getting the list # of layers for the image with the tag we used. (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry", - { "Cookie": cookie }) + { "Authorization": "Token " + token }) if data[0] != imagetagid: raise ValueError(["Expected first layer id '%s' to match image id '%s'", @@ -190,9 +192,9 @@ def download_template(name, server, destdir): if not os.path.exists(jsonfile) or not os.path.exists(datafile): # The '/json' URL gives us some metadata about the layer res = save_data(registryserver, "/v1/images/" + layerid + "/json", - { "Cookie": cookie }, jsonfile) + { "Authorization": "Token " + token }, jsonfile) createdFiles.append(jsonfile) - layersize = int(res.info().getheader("x-docker-size")) + layersize = int(res.info().getheader("Content-Length")) datacsum = None if layerid in checksums: @@ -201,7 +203,7 @@ def download_template(name, server, destdir): # and the '/layer' URL is the actual payload, provided # as a tar.gz archive save_data(registryserver, "/v1/images/" + layerid + "/layer", - { "Cookie": cookie }, datafile, datacsum, layersize) + { "Authorization": "Token " + token }, datafile, datacsum, layersize) createdFiles.append(datafile) # Strangely the 'json' data for a layer doesn't include -- 2.1.0

Any custom source provider can be added to virt-sandbox-image as a source --- virt-sandbox-image/sources/Source.py | 8 ++++++++ virt-sandbox-image/virt-sandbox-image.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 virt-sandbox-image/sources/Source.py diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py new file mode 100644 index 0000000..cfc75d3 --- /dev/null +++ b/virt-sandbox-image/sources/Source.py @@ -0,0 +1,8 @@ +#!/usr/bin/python + +from abc import ABCMeta, abstractmethod + +class Source(): + __metaclass__ = ABCMeta + def __init__(self): + pass diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 324e568..99ed46e 100644 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -1,5 +1,5 @@ #!/usr/bin/python -Es -# +# -*- coding: utf-8 -*- # Authors: Daniel P. Berrange <berrange(a)redhat.com&gt; # Eren Yagdiran <erenyagdiran(a)gmail.com&gt; # @@ -38,6 +38,34 @@ default_template_dir = "/var/lib/libvirt/templates" debug = True verbose = True +sys.dont_write_bytecode = True + + +##Hook mechanism starts## +import __builtin__ +from sources.Source import Source +__builtin__.hookHolder = {} +def add_hook(driverName,clazz): + holder = __builtin__.hookHolder + if not issubclass(clazz,Source): + raise Exception("Loading %s failed. Make sure it is a subclass Of %s" %(clazz,Source)) + holder[driverName] = clazz + +def init_from_name(name): + holder = __builtin__.hookHolder + return holder.get(name,None) + +__builtin__.add_hook = add_hook +__builtin__.init_from_name = init_from_name +from sources import * + +def dynamic_source_loader(name): + obj = init_from_name(name) + if obj == None: + raise IOError + return obj() +##Hook mechanism ends + gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") gettext.textdomain("libvirt-sandbox") try: -- 2.1.0

Refactor download function from virt-sandbox-image to use the newly introduced Source abstract class. The docker-specific download code is moved to a new DockerSource class. --- virt-sandbox-image/sources/DockerSource.py | 193 +++++++++++++++++++++++++++ virt-sandbox-image/sources/Source.py | 5 + virt-sandbox-image/virt-sandbox-image.py | 202 ++++------------------------- 3 files changed, 225 insertions(+), 175 deletions(-) create mode 100644 virt-sandbox-image/sources/DockerSource.py diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py new file mode 100644 index 0000000..5bcd613 --- /dev/null +++ b/virt-sandbox-image/sources/DockerSource.py @@ -0,0 +1,193 @@ +#!/usr/bin/python + +from Source import Source +import urllib2 +import sys +import json +import traceback +import os +import subprocess +import shutil + +class DockerSource(Source): + default_index_server = "index.docker.io" + default_template_dir = "/var/lib/libvirt/templates" + default_image_path = "/var/lib/libvirt/templates" + default_disk_format = "qcow2" + + www_auth_username = None + www_auth_password = None + + def __init__(self,server="index.docker.io",destdir="/var/lib/libvirt/templates"): + self.default_index_server = server + self.default_template_dir = destdir + + def download_template(self,**args): + name = args['name'] + registry = args['registry'] if args['registry'] is not None else self.default_index_server + username = args['username'] + password = args['password'] + templatedir = args['templatedir'] if args['templatedir'] is not None else self.default_template_dir + self.__download_template(name,registry,username,password,templatedir) + + def __download_template(self,name, server,username,password,destdir): + + if username is not None: + self.www_auth_username = username + self.www_auth_password = password + + tag = "latest" + offset = name.find(':') + if offset != -1: + tag = name[offset + 1:] + name = name[0:offset] + try: + (data, res) = self.__get_json(server, "/v1/repositories/" + name + "/images", + {"X-Docker-Token": "true"}) + except urllib2.HTTPError, e: + raise ValueError(["Image '%s' does not exist" % name]) + + registryserver = res.info().getheader('X-Docker-Endpoints') + token = res.info().getheader('X-Docker-Token') + checksums = {} + for layer in data: + pass + (data, res) = self.__get_json(registryserver, "/v1/repositories/" + name + "/tags", + { "Authorization": "Token " + token }) + + cookie = res.info().getheader('Set-Cookie') + + if not tag in data: + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)]) + imagetagid = data[tag] + + (data, res) = self.__get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry", + { "Authorization": "Token "+token }) + + if data[0] != imagetagid: + raise ValueError(["Expected first layer id '%s' to match image id '%s'", + data[0], imagetagid]) + + try: + createdFiles = [] + createdDirs = [] + + for layerid in data: + templatedir = destdir + "/" + layerid + if not os.path.exists(templatedir): + os.mkdir(templatedir) + createdDirs.append(templatedir) + + jsonfile = templatedir + "/template.json" + datafile = templatedir + "/template.tar.gz" + + if not os.path.exists(jsonfile) or not os.path.exists(datafile): + res = self.__save_data(registryserver, "/v1/images/" + layerid + "/json", + { "Authorization": "Token " + token }, jsonfile) + createdFiles.append(jsonfile) + + layersize = int(res.info().getheader("Content-Length")) + + datacsum = None + if layerid in checksums: + datacsum = checksums[layerid] + + self.__save_data(registryserver, "/v1/images/" + layerid + "/layer", + { "Authorization": "Token "+token }, datafile, datacsum, layersize) + createdFiles.append(datafile) + + index = { + "name": name, + } + + indexfile = destdir + "/" + imagetagid + "/index.json" + print("Index file " + indexfile) + with open(indexfile, "w") as f: + f.write(json.dumps(index)) + except Exception as e: + traceback.print_exc() + for f in createdFiles: + try: + os.remove(f) + except: + pass + for d in createdDirs: + try: + shutil.rmtree(d) + except: + pass + def __save_data(self,server, path, headers, dest, checksum=None, datalen=None): + try: + res = self.__get_url(server, path, headers) + + csum = None + if checksum is not None: + csum = hashlib.sha256() + + pattern = [".", "o", "O", "o"] + patternIndex = 0 + donelen = 0 + + with open(dest, "w") as f: + while 1: + buf = res.read(1024*64) + if not buf: + break + if csum is not None: + csum.update(buf) + f.write(buf) + + if datalen is not None: + donelen = donelen + len(buf) + debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( + pattern[patternIndex], (donelen/1024), (datalen/1024) + )) + patternIndex = (patternIndex + 1) % 4 + + debug("\x1b[K") + if csum is not None: + csumstr = "sha256:" + csum.hexdigest() + if csumstr != checksum: + debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) + os.remove(dest) + raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) + debug("OK\n") + return res + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + + def __get_url(self,server, path, headers): + url = "https://" + server + path + debug("Fetching %s..." % url) + + req = urllib2.Request(url=url) + if json: + req.add_header("Accept", "application/json") + for h in headers.keys(): + req.add_header(h, headers[h]) + + #www Auth header starts + if self.www_auth_username is not None: + base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '') + req.add_header("Authorization", "Basic %s" % base64string) + #www Auth header finish + + return urllib2.urlopen(req) + + def __get_json(self,server, path, headers): + try: + res = self.__get_url(server, path, headers) + data = json.loads(res.read()) + debug("OK\n") + return (data, res) + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + +def debug(msg): + sys.stderr.write(msg) + +if __name__ == "sources.DockerSource": + from __builtin__ import add_hook + add_hook('docker',DockerSource) diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py index cfc75d3..99f44bb 100644 --- a/virt-sandbox-image/sources/Source.py +++ b/virt-sandbox-image/sources/Source.py @@ -6,3 +6,8 @@ class Source(): __metaclass__ = ABCMeta def __init__(self): pass + + @abstractmethod + def download_template(self,**args): + pass + diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 99ed46e..1392e8f 100644 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -40,7 +40,6 @@ verbose = True sys.dont_write_bytecode = True - ##Hook mechanism starts## import __builtin__ from sources.Source import Source @@ -84,178 +83,6 @@ def debug(msg): def info(msg): sys.stdout.write(msg) -def get_url(server, path, headers): - url = "https://" + server + path - debug(" Fetching %s..." % url) - - req = urllib2.Request(url=url) - - if json: - req.add_header("Accept", "application/json") - - for h in headers.keys(): - req.add_header(h, headers[h]) - - return urllib2.urlopen(req) - -def get_json(server, path, headers): - try: - res = get_url(server, path, headers) - data = json.loads(res.read()) - debug("OK\n") - return (data, res) - except Exception, e: - debug("FAIL %s\n" % str(e)) - raise - -def save_data(server, path, headers, dest, checksum=None, datalen=None): - try: - res = get_url(server, path, headers) - - csum = None - if checksum is not None: - csum = hashlib.sha256() - - pattern = [".", "o", "O", "o"] - patternIndex = 0 - donelen = 0 - - with open(dest, "w") as f: - while 1: - buf = res.read(1024*64) - if not buf: - break - if csum is not None: - csum.update(buf) - f.write(buf) - - if datalen is not None: - donelen = donelen + len(buf) - debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( - pattern[patternIndex], (donelen/1024), (datalen/1024) - )) - patternIndex = (patternIndex + 1) % 4 - - debug("\x1b[K") - if csum is not None: - csumstr = "sha256:" + csum.hexdigest() - if csumstr != checksum: - debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) - os.remove(dest) - raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) - - debug("OK\n") - return res - except Exception, e: - debug("FAIL %s\n" % str(e)) - raise - - -def download_template(name, server, destdir): - tag = "latest" - - offset = name.find(':') - if offset != -1: - tag = name[offset + 1:] - name = name[0:offset] - - # First we must ask the index server about the image name. THe - # index server will return an auth token we can use when talking - # to the registry server. We need this token even when anonymous - try: - (data, res) = get_json(server, "/v1/repositories/" + name + "/images", - {"X-Docker-Token": "true"}) - except urllib2.HTTPError, e: - raise ValueError(["Image '%s' does not exist" % name]) - - registryserver = res.info().getheader('X-Docker-Endpoints') - token = res.info().getheader('X-Docker-Token') - checksums = {} - for layer in data: - pass - # XXX Checksums here don't appear to match the data in - # image download later. Find out what these are sums of - #checksums[layer["id"]] = layer["checksum"] - - # Now we ask the registry server for the list of tags associated - # with the image. Tags usually reflect some kind of version of - # the image, but they aren't officially "versions". There is - # always a "latest" tag which is the most recent upload - # - # We must pass in the auth token from the index server. This - # token can only be used once, and we're given a cookie back - # in return to use for later RPC calls. - (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags", - { "Authorization": "Token " + token }) - - cookie = res.info().getheader('Set-Cookie') - - if not tag in data: - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)]) - imagetagid = data[tag] - - # Only base images are self-contained, most images reference one - # or more parents, in a linear stack. Here we are getting the list - # of layers for the image with the tag we used. - (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry", - { "Authorization": "Token " + token }) - - if data[0] != imagetagid: - raise ValueError(["Expected first layer id '%s' to match image id '%s'", - data[0], imagetagid]) - - try: - createdFiles = [] - createdDirs = [] - - for layerid in data: - templatedir = destdir + "/" + layerid - if not os.path.exists(templatedir): - os.mkdir(templatedir) - createdDirs.append(templatedir) - - jsonfile = templatedir + "/template.json" - datafile = templatedir + "/template.tar.gz" - - if not os.path.exists(jsonfile) or not os.path.exists(datafile): - # The '/json' URL gives us some metadata about the layer - res = save_data(registryserver, "/v1/images/" + layerid + "/json", - { "Authorization": "Token " + token }, jsonfile) - createdFiles.append(jsonfile) - layersize = int(res.info().getheader("Content-Length")) - - datacsum = None - if layerid in checksums: - datacsum = checksums[layerid] - - # and the '/layer' URL is the actual payload, provided - # as a tar.gz archive - save_data(registryserver, "/v1/images/" + layerid + "/layer", - { "Authorization": "Token " + token }, datafile, datacsum, layersize) - createdFiles.append(datafile) - - # Strangely the 'json' data for a layer doesn't include - # its actual name, so we save that in a json file of our own - index = { - "name": name, - } - - indexfile = destdir + "/" + imagetagid + "/index.json" - with open(indexfile, "w") as f: - f.write(json.dumps(index)) - except Exception, e: - for f in createdFiles: - try: - os.remove(f) - except: - pass - for d in createdDirs: - try: - os.rmdir(d) - except: - pass - - def delete_template(name, destdir): imageusage = {} imageparent = {} @@ -360,8 +187,16 @@ def create_template(name, imagepath, format, destdir): def download(args): - info("Downloading %s from %s to %s\n" % (args.name, default_index_server, default_template_dir)) - download_template(args.name, default_index_server, default_template_dir) + try: + dynamic_source_loader(args.source).download_template(name=args.name, + registry=args.registry, + username=args.username, + password=args.password, + templatedir=args.template_dir) + except IOError,e: + print "Source %s cannot be found in given path" %args.source + except Exception,e: + print "Download Error %s" % str(e) def delete(args): info("Deleting %s from %s\n" % (args.name, default_template_dir)) @@ -375,10 +210,27 @@ def requires_name(parser): parser.add_argument("name", help=_("name of the template")) +def requires_source(parser): + parser.add_argument("-s","--source", + default="docker", + help=_("name of the template")) + +def requires_auth_conn(parser): + parser.add_argument("-r","--registry", + help=_("Url of the custom registry")) + parser.add_argument("-u","--username", + help=_("Username for the custom registry")) + parser.add_argument("-p","--password", + help=_("Password for the custom registry")) + parser.add_argument("-t","--template-dir", + help=_("Template directory for saving templates")) + def gen_download_args(subparser): parser = subparser.add_parser("download", help=_("Download template data")) + requires_source(parser) requires_name(parser) + requires_auth_conn(parser) parser.set_defaults(func=download) def gen_delete_args(subparser): -- 2.1.0

Move the docker-related code to the DockerSource and use the Source mechanism --- virt-sandbox-image/sources/DockerSource.py | 95 ++++++++++++++++++++++++++++++ virt-sandbox-image/sources/Source.py | 5 ++ virt-sandbox-image/virt-sandbox-image.py | 72 +++++----------------- 3 files changed, 115 insertions(+), 57 deletions(-) diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py index 5bcd613..f33f94b 100644 --- a/virt-sandbox-image/sources/DockerSource.py +++ b/virt-sandbox-image/sources/DockerSource.py @@ -185,6 +185,101 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise + def create_template(self,**args): + name = args['name'] + driver = args['driver'] + path = args['imagepath'] + path = path if path is not None else self.default_image_path + format = args['format'] + format = format if format is not None else self.default_disk_format + + self.__create_template(name, + driver, + path, + format, + path) + + def __create_template(self,name,driver,image_path,format,destdir): + self.__check_disk_format(format) + + imagelist = self.__get_image_list(name,destdir) + imagelist.reverse() + + parentImage = None + for imagetagid in imagelist: + templateImage = destdir + "/" + imagetagid + "/template." + format + cmd = ["qemu-img","create","-f","qcow2"] + if parentImage is not None: + cmd.append("-o") + cmd.append("backing_fmt=qcow2,backing_file=%s" % parentImage) + cmd.append(templateImage) + if parentImage is None: + cmd.append("10G") + subprocess.call(cmd) + + if parentImage is None: + self.__format_disk(templateImage,format,driver) + + self.__extract_tarballs(destdir + "/" + imagetagid + "/template.",format,driver) + parentImage = templateImage + + + def __check_disk_format(self,format): + supportedFormats = ['qcow2'] + if not format in supportedFormats: + raise ValueError(["Unsupported image format %s" % format]) + + def __get_image_list(self,name,destdir): + imageparent = {} + imagenames = {} + imagedirs = os.listdir(destdir) + for imagetagid in imagedirs: + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile,"r") as f: + index = json.load(f) + imagenames[index["name"]] = imagetagid + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + with open(jsonfile,"r") as f: + template = json.load(f) + parent = template.get("parent",None) + if parent: + imageparent[imagetagid] = parent + if not name in imagenames: + raise ValueError(["Image %s does not exist locally" %name]) + imagetagid = imagenames[name] + imagelist = [] + while imagetagid != None: + imagelist.append(imagetagid) + parent = imageparent.get(imagetagid,None) + imagetagid = parent + return imagelist + + def __format_disk(self,disk,format,driver): + cmd = ['virt-sandbox', + '-c',driver, + '--disk=file:disk_image=%s,format=%s' %(disk,format), + '/sbin/mkfs.ext3', + '/dev/disk/by-tag/disk_image'] + subprocess.call(cmd) + + def __extract_tarballs(self,directory,format,driver): + tempdir = "/mnt" + tarfile = directory + "tar.gz" + diskfile = directory + "qcow2" + cmd = ['virt-sandbox', + '-c',driver, + '-m', + 'host-image:/mnt=%s,format=%s' %(diskfile,format), + '--', + '/bin/tar', + 'zxvf', + '%s' %tarfile, + '-C', + '/mnt'] + subprocess.call(cmd) + def debug(msg): sys.stderr.write(msg) diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py index 99f44bb..89db14a 100644 --- a/virt-sandbox-image/sources/Source.py +++ b/virt-sandbox-image/sources/Source.py @@ -11,3 +11,8 @@ class Source(): def download_template(self,**args): pass + @abstractmethod + def create_template(self,**args): + pass + + diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 1392e8f..58055b3 100644 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -132,60 +132,6 @@ def delete_template(name, destdir): parent = None imagetagid = parent - -def get_image_list(name, destdir): - imageparent = {} - imagenames = {} - imagedirs = os.listdir(destdir) - for imagetagid in imagedirs: - indexfile = destdir + "/" + imagetagid + "/index.json" - if os.path.exists(indexfile): - with open(indexfile, "r") as f: - index = json.load(f) - imagenames[index["name"]] = imagetagid - jsonfile = destdir + "/" + imagetagid + "/template.json" - if os.path.exists(jsonfile): - with open(jsonfile, "r") as f: - template = json.load(f) - - parent = template.get("parent", None) - if parent: - imageparent[imagetagid] = parent - - if not name in imagenames: - raise ValueError(["Image %s does not exist locally" % name]) - - imagetagid = imagenames[name] - imagelist = [] - while imagetagid != None: - imagelist.append(imagetagid) - parent = imageparent.get(imagetagid, None) - imagetagid = parent - - return imagelist - -def create_template(name, imagepath, format, destdir): - if not format in ["qcow2"]: - raise ValueError(["Unsupported image format %s" % format]) - - imagelist = get_image_list(name, destdir) - imagelist.reverse() - - parentImage = None - for imagetagid in imagelist: - templateImage = destdir + "/" + imagetagid + "/template." + format - cmd = ["qemu-img", "create", "-f", "qcow2"] - if parentImage is not None: - cmd.append("-o") - cmd.append("backing_fmt=qcow2,backing_file=%s" % parentImage) - cmd.append(templateImage) - if parentImage is None: - cmd.append("10G") - debug("Run %s\n" % " ".join(cmd)) - subprocess.call(cmd) - parentImage = templateImage - - def download(args): try: dynamic_source_loader(args.source).download_template(name=args.name, @@ -203,8 +149,13 @@ def delete(args): delete_template(args.name, default_template_dir) def create(args): - info("Creating %s from %s in format %s\n" % (args.imagepath, args.name, args.format)) - create_template(args.name, args.imagepath, args.format, default_template_dir) + try: + dynamic_source_loader(args.source).create_template(name=args.name, + driver=args.driver, + imagepath=args.imagepath, + format=args.format) + except Exception,e: + print "Create Error %s" % str(e) def requires_name(parser): parser.add_argument("name", @@ -215,6 +166,11 @@ def requires_source(parser): default="docker", help=_("name of the template")) +def requires_driver(parser): + parser.add_argument("-d","--driver", + default="qemu:///session", + help=_("Type of the driver")) + def requires_auth_conn(parser): parser.add_argument("-r","--registry", help=_("Url of the custom registry")) @@ -243,10 +199,12 @@ def gen_create_args(subparser): parser = subparser.add_parser("create", help=_("Create image from template data")) requires_name(parser) + requires_source(parser) + requires_driver(parser) parser.add_argument("imagepath", help=_("path for image")) parser.add_argument("format", - help=_("format")) + help=_("format format for image")) parser.set_defaults(func=create) if __name__ == '__main__': -- 2.1.0

Refactoring delete function from virt-sandbox-image to DockerSource. Delete function can delete templates by name. --- virt-sandbox-image/sources/DockerSource.py | 53 +++++++++++++++++++++++++++ virt-sandbox-image/sources/Source.py | 3 ++ virt-sandbox-image/virt-sandbox-image.py | 59 ++++-------------------------- 3 files changed, 64 insertions(+), 51 deletions(-) diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py index f33f94b..180d3f2 100644 --- a/virt-sandbox-image/sources/DockerSource.py +++ b/virt-sandbox-image/sources/DockerSource.py @@ -280,6 +280,59 @@ class DockerSource(Source): '/mnt'] subprocess.call(cmd) + def delete_template(self,**args): + imageusage = {} + imageparent = {} + imagenames = {} + name = args['name'] + destdir = args['imagepath'] + destdir = destdir if destdir is not None else default_template_dir + imagedirs = os.listdir(destdir) + for imagetagid in imagedirs: + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile,"r") as f: + index = json.load(f) + imagenames[index["name"]] = imagetagid + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + with open(jsonfile,"r") as f: + template = json.load(f) + + parent = template.get("parent",None) + if parent: + if parent not in imageusage: + imageusage[parent] = [] + imageusage[parent].append(imagetagid) + imageparent[imagetagid] = parent + + + if not name in imagenames: + raise ValueError(["Image %s does not exist locally" %name]) + + imagetagid = imagenames[name] + while imagetagid != None: + debug("Remove %s\n" % imagetagid) + parent = imageparent.get(imagetagid,None) + + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + os.remove(indexfile) + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + os.remove(jsonfile) + datafile = destdir + "/" + imagetagid + "/template.tar.gz" + if os.path.exists(datafile): + os.remove(datafile) + imagedir = destdir + "/" + imagetagid + shutil.rmtree(imagedir) + + if parent: + if len(imageusage[parent]) != 1: + debug("Parent %s is shared\n" % parent) + parent = None + imagetagid = parent + def debug(msg): sys.stderr.write(msg) diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py index 89db14a..36c4343 100644 --- a/virt-sandbox-image/sources/Source.py +++ b/virt-sandbox-image/sources/Source.py @@ -15,4 +15,7 @@ class Source(): def create_template(self,**args): pass + @abstractmethod + def delete_template(self,**args): + pass diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index 58055b3..c320105 100644 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -83,55 +83,6 @@ def debug(msg): def info(msg): sys.stdout.write(msg) -def delete_template(name, destdir): - imageusage = {} - imageparent = {} - imagenames = {} - imagedirs = os.listdir(destdir) - for imagetagid in imagedirs: - indexfile = destdir + "/" + imagetagid + "/index.json" - if os.path.exists(indexfile): - with open(indexfile, "r") as f: - index = json.load(f) - imagenames[index["name"]] = imagetagid - jsonfile = destdir + "/" + imagetagid + "/template.json" - if os.path.exists(jsonfile): - with open(jsonfile, "r") as f: - template = json.load(f) - - parent = template.get("parent", None) - if parent: - if parent not in imageusage: - imageusage[parent] = [] - imageusage[parent].append(imagetagid) - imageparent[imagetagid] = parent - - if not name in imagenames: - raise ValueError(["Image %s does not exist locally" % name]) - - imagetagid = imagenames[name] - while imagetagid != None: - debug("Remove %s\n" % imagetagid) - parent = imageparent.get(imagetagid, None) - - indexfile = destdir + "/" + imagetagid + "/index.json" - if os.path.exists(indexfile): - os.remove(indexfile) - jsonfile = destdir + "/" + imagetagid + "/template.json" - if os.path.exists(jsonfile): - os.remove(jsonfile) - datafile = destdir + "/" + imagetagid + "/template.tar.gz" - if os.path.exists(datafile): - os.remove(datafile) - imagedir = destdir + "/" + imagetagid - os.rmdir(imagedir) - - if parent: - if len(imageusage[parent]) != 1: - debug("Parent %s is shared\n" % parent) - parent = None - imagetagid = parent - def download(args): try: dynamic_source_loader(args.source).download_template(name=args.name, @@ -145,8 +96,11 @@ def download(args): print "Download Error %s" % str(e) def delete(args): - info("Deleting %s from %s\n" % (args.name, default_template_dir)) - delete_template(args.name, default_template_dir) + try: + dynamic_source_loader(args.source).delete_template(name=args.name, + imagepath=args.imagepath) + except Exception,e: + print "Delete Error %s", str(e) def create(args): try: @@ -193,6 +147,9 @@ def gen_delete_args(subparser): parser = subparser.add_parser("delete", help=_("Delete template data")) requires_name(parser) + requires_source(parser) + parser.add_argument("imagepath", + help=_("Path for image")) parser.set_defaults(func=delete) def gen_create_args(subparser): -- 2.1.0

Commandline parameters for running a template --- virt-sandbox-image/virt-sandbox-image.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py index c320105..81b825c 100644 --- a/virt-sandbox-image/virt-sandbox-image.py +++ b/virt-sandbox-image/virt-sandbox-image.py @@ -164,6 +164,18 @@ def gen_create_args(subparser): help=_("format format for image")) parser.set_defaults(func=create) +def gen_run_args(subparser): + parser = subparser.add_parser("run", + help=_("Run a already built image")) + requires_name(parser) + requires_source(parser) + requires_driver(parser) + parser.add_argument("imagepath", + help=_("path for image")) + parser.add_argument("-c","--command", + help=_("Igniter command for image")) + parser.set_defaults(func=run) + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Sandbox Container Image Tool') @@ -171,6 +183,7 @@ if __name__ == '__main__': gen_download_args(subparser) gen_delete_args(subparser) gen_create_args(subparser) + gen_run_args(subparser) try: args = parser.parse_args() -- 2.1.0

Provide a way to know which disk image to use for the sandbox depending on the used source DockerSource will need to locate the topmost disk image among all the layers images --- virt-sandbox-image/sources/DockerSource.py | 9 +++++++++ virt-sandbox-image/sources/Source.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py index 4326624..01ef901 100644 --- a/virt-sandbox-image/sources/DockerSource.py +++ b/virt-sandbox-image/sources/DockerSource.py @@ -342,6 +342,15 @@ class DockerSource(Source): parent = None imagetagid = parent + def get_disk(self,**args): + name = args['name'] + destdir = args['path'] + imageList = self.__get_image_list(name,destdir) + toplayer = imageList[0] + diskfile = destdir + "/" + toplayer + "/template.qcow2" + configfile = destdir + "/" + toplayer + "/template.json" + return (diskfile,configfile) + def get_command(self,configfile): configParser = DockerConfParser(configfile) commandToRun = configParser.getRunCommand() diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py index c4087ca..739de78 100644 --- a/virt-sandbox-image/sources/Source.py +++ b/virt-sandbox-image/sources/Source.py @@ -23,3 +23,5 @@ class Source(): def get_command(self,**args): pass + def get_disk(self,**args): + pass -- 2.1.0