[libvirt] [PATCH sandbox v5 01/20] Add virt-sandbox-image

From: Daniel P Berrange <berrange(a)redhat.com&gt; virt-sandbox-image.py is a python script that lets you download Docker images easily. It is a proof of concept code and consumes Docker Rest API. Signed-off-by: Daniel P. Berrange <berrange(a)redhat.com&gt; --- bin/Makefile.am | 3 +- bin/virt-sandbox-image | 8 + configure.ac | 2 + libvirt-sandbox.spec.in | 2 + libvirt-sandbox/Makefile.am | 2 +- libvirt-sandbox/image/Makefile.am | 8 + libvirt-sandbox/image/__init__.py | 0 libvirt-sandbox/image/cli.py | 394 ++++++++++++++++++++++++++++++++++++++ po/POTFILES.in | 1 + 9 files changed, 418 insertions(+), 2 deletions(-) create mode 100644 bin/virt-sandbox-image create mode 100644 libvirt-sandbox/image/Makefile.am create mode 100644 libvirt-sandbox/image/__init__.py create mode 100644 libvirt-sandbox/image/cli.py diff --git a/bin/Makefile.am b/bin/Makefile.am index 416f86f..deedcf6 100644 --- a/bin/Makefile.am +++ b/bin/Makefile.am @@ -3,7 +3,8 @@ bin_PROGRAMS = virt-sandbox libexec_PROGRAMS = virt-sandbox-service-util -bin_SCRIPTS = virt-sandbox-service +bin_SCRIPTS = virt-sandbox-service \ + virt-sandbox-image virtsandboxcompdir = $(datarootdir)/bash-completion/completions/ diff --git a/bin/virt-sandbox-image b/bin/virt-sandbox-image new file mode 100644 index 0000000..7e0d76b --- /dev/null +++ b/bin/virt-sandbox-image @@ -0,0 +1,8 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +from libvirt_sandbox.image import cli +import sys + +if __name__ == '__main__': + sys.exit(cli.main()) diff --git a/configure.ac b/configure.ac index 71c4392..a8376ca 100644 --- a/configure.ac +++ b/configure.ac @@ -124,10 +124,12 @@ dnl Should be in m4/virt-gettext.m4 but intltoolize is too dnl dumb to find it there IT_PROG_INTLTOOL([0.35.0]) +AM_PATH_PYTHON AC_OUTPUT(Makefile libvirt-sandbox/Makefile libvirt-sandbox/tests/Makefile + libvirt-sandbox/image/Makefile bin/Makefile examples/Makefile docs/Makefile diff --git a/libvirt-sandbox.spec.in b/libvirt-sandbox.spec.in index 4978242..54fde55 100644 --- a/libvirt-sandbox.spec.in +++ b/libvirt-sandbox.spec.in @@ -98,7 +98,9 @@ rm -rf $RPM_BUILD_ROOT %dir %{_sysconfdir}/libvirt-sandbox/services %{_bindir}/virt-sandbox %{_bindir}/virt-sandbox-service +%{_bindir}/virt-sandbox-image %{_libexecdir}/virt-sandbox-service-util +%{python_sitelib}/libvirt_sandbox %{_mandir}/man1/virt-sandbox.1* %{_mandir}/man1/virt-sandbox-service.1* %{_mandir}/man1/virt-sandbox-service-*.1* diff --git a/libvirt-sandbox/Makefile.am b/libvirt-sandbox/Makefile.am index 597803e..b303078 100644 --- a/libvirt-sandbox/Makefile.am +++ b/libvirt-sandbox/Makefile.am @@ -2,7 +2,7 @@ EXTRA_DIST = libvirt-sandbox.sym CLEANFILES = -SUBDIRS = tests +SUBDIRS = tests image rundir = $(localstatedir)/run diff --git a/libvirt-sandbox/image/Makefile.am b/libvirt-sandbox/image/Makefile.am new file mode 100644 index 0000000..7c8da51 --- /dev/null +++ b/libvirt-sandbox/image/Makefile.am @@ -0,0 +1,8 @@ + +pythonsandboxdir = $(pythondir)/libvirt_sandbox +pythonsandbox_DATA = __init__.py + +pythonimagedir = $(pythondir)/libvirt_sandbox/image +pythonimage_DATA = __init__.py cli.py + +EXTRA_DIST = $(pythonimage_DATA) diff --git a/libvirt-sandbox/image/__init__.py b/libvirt-sandbox/image/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py new file mode 100644 index 0000000..ec96c7e --- /dev/null +++ b/libvirt-sandbox/image/cli.py @@ -0,0 +1,394 @@ +#!/usr/bin/python -Es +# +# Authors: Daniel P. Berrange <berrange(a)redhat.com&gt; +# +# Copyright (C) 2013 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# + +import argparse +import gettext +import hashlib +import json +import os +import os.path +import shutil +import sys +import urllib2 +import subprocess + +default_index_server = "index.docker.io" +default_template_dir = "/var/lib/libvirt/templates" + +debug = True +verbose = True + +gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") +gettext.textdomain("libvirt-sandbox") +try: + gettext.install("libvirt-sandbox", + localedir="/usr/share/locale", + unicode=False, + codeset = 'utf-8') +except IOError: + import __builtin__ + __builtin__.__dict__['_'] = unicode + + +def debug(msg): + sys.stderr.write(msg) + +def info(msg): + sys.stdout.write(msg) + +def get_url(server, path, headers): + url = "https://" + server + path + debug(" Fetching %s..." % url) + req = urllib2.Request(url=url) + + if json: + req.add_header("Accept", "application/json") + + for h in headers.keys(): + req.add_header(h, headers[h]) + + return urllib2.urlopen(req) + +def get_json(server, path, headers): + try: + res = get_url(server, path, headers) + data = json.loads(res.read()) + debug("OK\n") + return (data, res) + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + +def save_data(server, path, headers, dest, checksum=None, datalen=None): + try: + res = get_url(server, path, headers) + + csum = None + if checksum is not None: + csum = hashlib.sha256() + + pattern = [".", "o", "O", "o"] + patternIndex = 0 + donelen = 0 + + with open(dest, "w") as f: + while 1: + buf = res.read(1024*64) + if not buf: + break + if csum is not None: + csum.update(buf) + f.write(buf) + + if datalen is not None: + donelen = donelen + len(buf) + debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( + pattern[patternIndex], (donelen/1024), (datalen/1024) + )) + patternIndex = (patternIndex + 1) % 4 + + debug("\x1b[K") + if csum is not None: + csumstr = "sha256:" + csum.hexdigest() + if csumstr != checksum: + debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) + os.remove(dest) + raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) + debug("OK\n") + return res + except Exception, e: + debug("FAIL %s\n" % str(e)) + raise + + +def download_template(name, server, destdir): + tag = "latest" + + offset = name.find(':') + if offset != -1: + tag = name[offset + 1:] + name = name[0:offset] + + # First we must ask the index server about the image name. THe + # index server will return an auth token we can use when talking + # to the registry server. We need this token even when anonymous + try: + (data, res) = get_json(server, "/v1/repositories/" + name + "/images", + {"X-Docker-Token": "true"}) + except urllib2.HTTPError, e: + raise ValueError(["Image '%s' does not exist" % name]) + + registryserver = res.info().getheader('X-Docker-Endpoints') + token = res.info().getheader('X-Docker-Token') + checksums = {} + for layer in data: + pass + # XXX Checksums here don't appear to match the data in + # image download later. Find out what these are sums of + #checksums[layer["id"]] = layer["checksum"] + + # Now we ask the registry server for the list of tags associated + # with the image. Tags usually reflect some kind of version of + # the image, but they aren't officially "versions". There is + # always a "latest" tag which is the most recent upload + # + # We must pass in the auth token from the index server. This + # token can only be used once, and we're given a cookie back + # in return to use for later RPC calls. + (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags", + { "Authorization": "Token " + token }) + + cookie = res.info().getheader('Set-Cookie') + + if not tag in data: + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)]) + imagetagid = data[tag] + + # Only base images are self-contained, most images reference one + # or more parents, in a linear stack. Here we are getting the list + # of layers for the image with the tag we used. + (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry", + { "Cookie": cookie }) + + if data[0] != imagetagid: + raise ValueError(["Expected first layer id '%s' to match image id '%s'", + data[0], imagetagid]) + + try: + createdFiles = [] + createdDirs = [] + + for layerid in data: + templatedir = destdir + "/" + layerid + if not os.path.exists(templatedir): + os.mkdir(templatedir) + createdDirs.append(templatedir) + + jsonfile = templatedir + "/template.json" + datafile = templatedir + "/template.tar.gz" + + if not os.path.exists(jsonfile) or not os.path.exists(datafile): + # The '/json' URL gives us some metadata about the layer + res = save_data(registryserver, "/v1/images/" + layerid + "/json", + { "Cookie": cookie }, jsonfile) + createdFiles.append(jsonfile) + layersize = int(res.info().getheader("x-docker-size")) + + datacsum = None + if layerid in checksums: + datacsum = checksums[layerid] + + # and the '/layer' URL is the actual payload, provided + # as a tar.gz archive + save_data(registryserver, "/v1/images/" + layerid + "/layer", + { "Cookie": cookie }, datafile, datacsum, layersize) + createdFiles.append(datafile) + + # Strangely the 'json' data for a layer doesn't include + # its actual name, so we save that in a json file of our own + index = { + "name": name, + } + + indexfile = destdir + "/" + imagetagid + "/index.json" + with open(indexfile, "w") as f: + f.write(json.dumps(index)) + except Exception, e: + for f in createdFiles: + try: + os.remove(f) + except: + pass + for d in createdDirs: + try: + os.rmdir(d) + except: + pass + + +def delete_template(name, destdir): + imageusage = {} + imageparent = {} + imagenames = {} + imagedirs = os.listdir(destdir) + for imagetagid in imagedirs: + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile, "r") as f: + index = json.load(f) + imagenames[index["name"]] = imagetagid + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + with open(jsonfile, "r") as f: + template = json.load(f) + + parent = template.get("parent", None) + if parent: + if parent not in imageusage: + imageusage[parent] = [] + imageusage[parent].append(imagetagid) + imageparent[imagetagid] = parent + + if not name in imagenames: + raise ValueError(["Image %s does not exist locally" % name]) + + imagetagid = imagenames[name] + while imagetagid != None: + debug("Remove %s\n" % imagetagid) + parent = imageparent.get(imagetagid, None) + + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + os.remove(indexfile) + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + os.remove(jsonfile) + datafile = destdir + "/" + imagetagid + "/template.tar.gz" + if os.path.exists(datafile): + os.remove(datafile) + imagedir = destdir + "/" + imagetagid + os.rmdir(imagedir) + + if parent: + if len(imageusage[parent]) != 1: + debug("Parent %s is shared\n" % parent) + parent = None + imagetagid = parent + + +def get_image_list(name, destdir): + imageparent = {} + imagenames = {} + imagedirs = os.listdir(destdir) + for imagetagid in imagedirs: + indexfile = destdir + "/" + imagetagid + "/index.json" + if os.path.exists(indexfile): + with open(indexfile, "r") as f: + index = json.load(f) + imagenames[index["name"]] = imagetagid + jsonfile = destdir + "/" + imagetagid + "/template.json" + if os.path.exists(jsonfile): + with open(jsonfile, "r") as f: + template = json.load(f) + + parent = template.get("parent", None) + if parent: + imageparent[imagetagid] = parent + + if not name in imagenames: + raise ValueError(["Image %s does not exist locally" % name]) + + imagetagid = imagenames[name] + imagelist = [] + while imagetagid != None: + imagelist.append(imagetagid) + parent = imageparent.get(imagetagid, None) + imagetagid = parent + + return imagelist + +def create_template(name, imagepath, format, destdir): + if not format in ["qcow2"]: + raise ValueError(["Unsupported image format %s" % format]) + + imagelist = get_image_list(name, destdir) + imagelist.reverse() + + parentImage = None + for imagetagid in imagelist: + templateImage = destdir + "/" + imagetagid + "/template." + format + cmd = ["qemu-img", "create", "-f", "qcow2"] + if parentImage is not None: + cmd.append("-o") + cmd.append("backing_fmt=qcow2,backing_file=%s" % parentImage) + cmd.append(templateImage) + if parentImage is None: + cmd.append("10G") + debug("Run %s\n" % " ".join(cmd)) + subprocess.call(cmd) + parentImage = templateImage + +def download(args): + info("Downloading %s from %s to %s\n" % (args.name, default_index_server, default_template_dir)) + download_template(args.name, default_index_server, default_template_dir) + +def delete(args): + info("Deleting %s from %s\n" % (args.name, default_template_dir)) + delete_template(args.name, default_template_dir) + +def create(args): + info("Creating %s from %s in format %s\n" % (args.imagepath, args.name, args.format)) + create_template(args.name, args.imagepath, args.format, default_template_dir) + +def requires_name(parser): + parser.add_argument("name", + help=_("name of the template")) + +def gen_download_args(subparser): + parser = subparser.add_parser("download", + help=_("Download template data")) + requires_name(parser) + parser.set_defaults(func=download) + +def gen_delete_args(subparser): + parser = subparser.add_parser("delete", + help=_("Delete template data")) + requires_name(parser) + parser.set_defaults(func=delete) + +def gen_create_args(subparser): + parser = subparser.add_parser("create", + help=_("Create image from template data")) + requires_name(parser) + parser.add_argument("imagepath", + help=_("path for image")) + parser.add_argument("format", + help=_("format")) + parser.set_defaults(func=create) + +def main(): + parser = argparse.ArgumentParser(description='Sandbox Container Image Tool') + + subparser = parser.add_subparsers(help=_("commands")) + gen_download_args(subparser) + gen_delete_args(subparser) + gen_create_args(subparser) + + try: + args = parser.parse_args() + args.func(args) + sys.exit(0) + except KeyboardInterrupt, e: + sys.exit(0) + except ValueError, e: + for line in e: + for l in line: + sys.stderr.write("%s: %s\n" % (sys.argv[0], l)) + sys.stderr.flush() + sys.exit(1) + except IOError, e: + sys.stderr.write("%s: %s: %s\n" % (sys.argv[0], e.filename, e.reason)) + sys.stderr.flush() + sys.exit(1) + except OSError, e: + sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) + sys.stderr.flush() + sys.exit(1) diff --git a/po/POTFILES.in b/po/POTFILES.in index afcb050..724c49c 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -11,3 +11,4 @@ libvirt-sandbox/libvirt-sandbox-context-interactive.c libvirt-sandbox/libvirt-sandbox-init-common.c libvirt-sandbox/libvirt-sandbox-rpcpacket.c libvirt-sandbox/libvirt-sandbox-util.c +libvirt-sandbox/image/cli.py -- 2.4.3