Re: [libvirt] [PATCH sandbox 1/1] Delete virt-sandbox-image tool now in separate repo

The virt-sandbox-image tool and its supporting code has been split into a separate libvirt-sandbox-image GIT repository. This allows its build system and distribution to work in the normal python way, and have a release lifecycle independent of the main libvirt-sandbox package. https://libvirt.org/git/?p=libvirt-sandbox-image.git;a=summary

Signed-off-by: Daniel P. Berrangé <berrange(a)redhat.com&gt; --- autobuild.sh | 3 +- bin/virt-sandbox-image | 8 - configure.ac | 4 - libvirt-sandbox.spec.in | 9 - libvirt-sandbox/image/Makefile.am | 10 - libvirt-sandbox/image/__init__.py | 0 libvirt-sandbox/image/cli.py | 282 ----------- libvirt-sandbox/image/sources/Makefile.am | 10 - libvirt-sandbox/image/sources/__init__.py | 0 libvirt-sandbox/image/sources/base.py | 160 ------- libvirt-sandbox/image/sources/docker.py | 690 --------------------------- libvirt-sandbox/image/sources/virtbuilder.py | 109 ----- libvirt-sandbox/image/template.py | 133 ------ m4/virt-win32.m4 | 5 - 14 files changed, 1 insertion(+), 1422 deletions(-) delete mode 100755 bin/virt-sandbox-image delete mode 100644 libvirt-sandbox/image/Makefile.am delete mode 100644 libvirt-sandbox/image/__init__.py delete mode 100644 libvirt-sandbox/image/cli.py delete mode 100644 libvirt-sandbox/image/sources/Makefile.am delete mode 100644 libvirt-sandbox/image/sources/__init__.py delete mode 100644 libvirt-sandbox/image/sources/base.py delete mode 100755 libvirt-sandbox/image/sources/docker.py delete mode 100755 libvirt-sandbox/image/sources/virtbuilder.py delete mode 100644 libvirt-sandbox/image/template.py diff --git a/autobuild.sh b/autobuild.sh index 9cd4ca2..c176fea 100755 --- a/autobuild.sh +++ b/autobuild.sh @@ -54,8 +54,7 @@ if [ -x /usr/bin/i686-pc-mingw32-gcc ]; then ../configure \ --build=$(uname -m)-pc-linux \ --host=i686-pc-mingw32 \ - --prefix="$AUTOBUILD_INSTALL_ROOT/i686-pc-mingw32/sys-root/mingw" \ - --without-python + --prefix="$AUTOBUILD_INSTALL_ROOT/i686-pc-mingw32/sys-root/mingw" make make install diff --git a/bin/virt-sandbox-image b/bin/virt-sandbox-image deleted file mode 100755 index 61346ef..0000000 --- a/bin/virt-sandbox-image +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -from libvirt_sandbox.image import cli -import sys - -if __name__ == '__main__': - sys.exit(cli.main()) diff --git a/configure.ac b/configure.ac index 28305d2..19d4523 100644 --- a/configure.ac +++ b/configure.ac @@ -130,13 +130,9 @@ dnl Should be in m4/virt-gettext.m4 but intltoolize is too dnl dumb to find it there IT_PROG_INTLTOOL([0.35.0]) -AM_PATH_PYTHON([3]) - AC_OUTPUT(Makefile libvirt-sandbox/Makefile libvirt-sandbox/tests/Makefile - libvirt-sandbox/image/Makefile - libvirt-sandbox/image/sources/Makefile bin/Makefile examples/Makefile docs/Makefile diff --git a/libvirt-sandbox.spec.in b/libvirt-sandbox.spec.in index 125a361..0374d9e 100644 --- a/libvirt-sandbox.spec.in +++ b/libvirt-sandbox.spec.in @@ -36,14 +36,7 @@ BuildRequires: zlib-devel >= 1.2.0, zlib-static BuildRequires: libtirpc-devel BuildRequires: rpcgen %endif -Requires: python3-rpm -# For virsh lxc-enter-namespace command -Requires: libvirt-client >= %{libvirt_version} -Requires: systemd >= 198 -Requires: pygobject3-base -Requires: libselinux-python3 Requires: %{name}-libs = %{version}-%{release} -Requires: %{_bindir}/virt-builder %package libs Group: Development/Libraries @@ -102,8 +95,6 @@ rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root,-) %{_bindir}/virt-sandbox -%{_bindir}/virt-sandbox-image -%{python3_sitelib}/libvirt_sandbox %{_mandir}/man1/virt-sandbox.1* %files libs -f %{name}.lang diff --git a/libvirt-sandbox/image/Makefile.am b/libvirt-sandbox/image/Makefile.am deleted file mode 100644 index 60afbf3..0000000 --- a/libvirt-sandbox/image/Makefile.am +++ /dev/null @@ -1,10 +0,0 @@ - -SUBDIRS = sources - -pythonsandboxdir = $(pythondir)/libvirt_sandbox -pythonsandbox_DATA = __init__.py - -pythonimagedir = $(pythondir)/libvirt_sandbox/image -pythonimage_DATA = __init__.py cli.py template.py - -EXTRA_DIST = $(pythonimage_DATA) diff --git a/libvirt-sandbox/image/__init__.py b/libvirt-sandbox/image/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py deleted file mode 100644 index 605183c..0000000 --- a/libvirt-sandbox/image/cli.py +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Authors: Daniel P. Berrange <berrange(a)redhat.com&gt; -# Eren Yagdiran <erenyagdiran(a)gmail.com&gt; -# -# Copyright (C) 2013-2015 Red Hat, Inc. -# Copyright (C) 2015 Universitat Politècnica de Catalunya. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -# - -import argparse -import gettext -import hashlib -import json -import os -import os.path -import re -import shutil -import sys -import subprocess -import random -import string - -from libvirt_sandbox.image import template - -if os.geteuid() == 0: - default_template_dir = "/var/lib/libvirt/templates" - default_image_dir = "/var/lib/libvirt/images" -else: - default_template_dir = os.environ['HOME'] + "/.local/share/libvirt/templates" - default_image_dir = os.environ['HOME'] + "/.local/share/libvirt/images" - -debug = False -verbose = False - -gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") -gettext.textdomain("libvirt-sandbox") -try: - gettext.install("libvirt-sandbox", - localedir="/usr/share/locale", - codeset = 'utf-8') -except IOError: - import __builtin__ - __builtin__.__dict__['_'] = unicode - - -def debug(msg): - sys.stderr.write(msg) - -def info(msg): - sys.stdout.write(msg) - -def get_template_dir(args): - tmpl = template.Template.from_uri(args.template) - return "%s/%s" % (args.template_dir, tmpl.source) - -def purge(args): - tmpl = template.Template.from_uri(args.template) - source = tmpl.get_source_impl() - source.delete_template(template=tmpl, - templatedir=get_template_dir(args)) - -def prepare(args): - tmpl = template.Template.from_uri(args.template) - source = tmpl.get_source_impl() - source.create_template(template=tmpl, - templatedir=get_template_dir(args), - connect=args.connect) - -def random_domain_name(tmpl): - randomid = ''.join(random.choice(string.ascii_lowercase) for i in range(10)) - return re.sub('[^a-z0-9-]', '_', tmpl.path[1:], re.I) + ":" + randomid - -def run(args): - if args.connect is not None: - check_connect(args.connect) - - tmpl = template.Template.from_uri(args.template) - source = tmpl.get_source_impl() - template_dir = get_template_dir(args) - - # Create the template image if needed - if not source.has_template(tmpl, template_dir): - prepare(args) - - name = args.name - if name is None: - name = random_domain_name(tmpl) - - diskfile = source.get_disk(template=tmpl, - templatedir=template_dir, - imagedir=args.image_dir, - sandboxname=name) - - commandToRun = source.get_command(tmpl, template_dir, args.args) - if len(commandToRun) == 0: - commandToRun = ["/bin/sh"] - cmd = ['virt-sandbox', '--name', name] - if args.connect is not None: - cmd.append("-c") - cmd.append(args.connect) - params = ['-m','host-image:/=%s,format=qcow2' % diskfile] - - networkArgs = args.network - if networkArgs is not None: - params.append('-N') - params.append(networkArgs) - - allEnvs = source.get_env(tmpl, template_dir) - envArgs = args.env - if envArgs is not None: - allEnvs = allEnvs + envArgs - for env in allEnvs: - envsplit = env.split("=") - envlen = len(envsplit) - if envlen == 2: - params.append("--env") - params.append(env) - else: - pass - - cmd = cmd + params + ['--'] + commandToRun - subprocess.call(cmd) - os.unlink(diskfile) - source.post_run(tmpl, template_dir, name) - -def list_cached(args): - tmpls = [] - if args.source is not None: - tmpls.extend(template.Template.get_all(args.source, - "%s/%s" % (args.template_dir, args.source))) - else: - for source in ["docker", "virt-builder"]: - tmpls.extend(template.Template.get_all(source, - "%s/%s" % (args.template_dir, source))) - for tmpl in tmpls: - print (tmpl) - -def requires_template(parser): - parser.add_argument("template", - help=_("URI of the template")) - -def requires_name(parser): - parser.add_argument("-n","--name", - help=_("Name of the running sandbox")) - -def requires_debug(parser): - parser.add_argument("-d","--debug", - default=False, action="store_true", - help=_("Run in debug mode")) - -def check_connect(connectstr): - supportedDrivers = ['lxc:///','qemu:///session','qemu:///system'] - if not connectstr in supportedDrivers: - raise ValueError("URI '%s' is not supported by virt-sandbox-image" % connectstr) - return True - -def requires_connect(parser): - parser.add_argument("-c","--connect", - help=_("Connect string for libvirt")) - -def requires_template_dir(parser): - global default_template_dir - parser.add_argument("-t","--template-dir", - default=default_template_dir, - help=_("Template directory for saving templates")) - -def requires_image_dir(parser): - global default_image_dir - parser.add_argument("-I","--image-dir", - default=default_image_dir, - help=_("Image directory for saving images")) - -def gen_command_parser(subparser, name, helptext): - parser = subparser.add_parser( - name, help=helptext, - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" - -Example supported URI formats: - - docker:///ubuntu?tag=15.04 - docker://username:password@index.docker.io/private/image - docker://registry.access.redhat.com/rhel6 - virt-builder:///fedora-20 -""") - return parser - -def gen_purge_args(subparser): - parser = gen_command_parser(subparser, "purge", - _("Purge cached template")) - requires_template(parser) - requires_template_dir(parser) - parser.set_defaults(func=purge) - -def gen_prepare_args(subparser): - parser = gen_command_parser(subparser, "prepare", - _("Prepare local template")) - requires_template(parser) - requires_connect(parser) - requires_template_dir(parser) - parser.set_defaults(func=prepare) - -def gen_run_args(subparser): - parser = gen_command_parser(subparser, "run", - _("Run an instance of a template")) - requires_name(parser) - requires_template(parser) - requires_connect(parser) - requires_template_dir(parser) - requires_image_dir(parser) - parser.add_argument("args", - nargs=argparse.REMAINDER, - help=_("command arguments to run")) - parser.add_argument("-N","--network", - help=_("Network params for running template")) - parser.add_argument("-e","--env",action="append", - help=_("Environment params for running template")) - - parser.set_defaults(func=run) - -def gen_list_args(subparser): - parser = gen_command_parser(subparser, "list", - _("List locally cached images")) - requires_template_dir(parser) - - parser.add_argument("-s","--source", - help=_("Name of the template source")) - - parser.set_defaults(func=list_cached) - -def main(): - parser = argparse.ArgumentParser(description="Sandbox Container Image Tool") - - requires_debug(parser) - - subparser = parser.add_subparsers(help=_("commands")) - subparser.required = True - subparser.dest = "command" - gen_purge_args(subparser) - gen_prepare_args(subparser) - gen_run_args(subparser) - gen_list_args(subparser) - - args = parser.parse_args() - if args.debug: - args.func(args) - sys.exit(0) - else: - try: - args.func(args) - sys.exit(0) - except KeyboardInterrupt as e: - sys.exit(0) - except ValueError as e: - sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) - sys.stderr.flush() - sys.exit(1) - except IOError as e: - sys.stderr.write("%s: %s\n" % (sys.argv[0], e.filename)) - sys.stderr.flush() - sys.exit(1) - except OSError as e: - sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) - sys.stderr.flush() - sys.exit(1) - except Exception as e: - print (e) - sys.exit(1) diff --git a/libvirt-sandbox/image/sources/Makefile.am b/libvirt-sandbox/image/sources/Makefile.am deleted file mode 100644 index 817baa0..0000000 --- a/libvirt-sandbox/image/sources/Makefile.am +++ /dev/null @@ -1,10 +0,0 @@ - -pythonimagedir = $(pythondir)/libvirt_sandbox/image/sources -pythonimage_DATA = \ - __init__.py \ - base.py \ - docker.py \ - virtbuilder.py \ - $(NULL) - -EXTRA_DIST = $(pythonimage_DATA) diff --git a/libvirt-sandbox/image/sources/__init__.py b/libvirt-sandbox/image/sources/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/libvirt-sandbox/image/sources/base.py b/libvirt-sandbox/image/sources/base.py deleted file mode 100644 index 0fc9243..0000000 --- a/libvirt-sandbox/image/sources/base.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2015 Universitat Politècnica de Catalunya. -# Copyright (C) 2015 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Author: Eren Yagdiran <erenyagdiran(a)gmail.com&gt; - -from abc import ABCMeta, abstractmethod -import subprocess - -class Source(): - '''The Source class defines the base interface for - all image provider source implementations. An image - provide source is able to download templates from - a repository, convert them to a host specific image - format and report commands used to invoke them.''' - - __metaclass__ = ABCMeta - def __init__(self): - pass - - @abstractmethod - def list_templates(self, templatedir): - """ - :param templatedir: local directory path in which to store the template - - Get a list of all templates that are locally cached - - :returns: a list of libvirt_sandbox.template.Template objects - """ - pass - - @abstractmethod - def has_template(self, template, templatedir): - """ - :param template: libvirt_sandbox.template.Template object - :param templatedir: local directory path in which to store the template - - Check if a template has already been created. - """ - pass - - @abstractmethod - def create_template(self, template, templatedir, - connect=None): - """ - :param template: libvirt_sandbox.template.Template object - :param templatedir: local directory path in which to store the template - :param connect: libvirt connection URI - - Create a set of local disk images populated with the content - of a template. The images creation process will be isolated - inside a sandbox using the requested libvirt connection URI. - """ - pass - - @abstractmethod - def delete_template(self, template, templatedir): - """ - :param template: libvirt_sandbox.template.Template object - :param templatedir: local directory path from which to delete template - - Delete all local files associated with the template - """ - pass - - @abstractmethod - def get_command(self, template, templatedir, userargs): - """ - :param template: libvirt_sandbox.template.Template object - :param templatedir: local directory path in which templates are stored - :param userargs: user specified arguments to run - - Get the command line to invoke in the container. If userargs - is specified, then this should override the default args in - the image""" - pass - - @abstractmethod - def get_disk(self, template, templatedir, imagedir, sandboxname): - """ - :param template: libvirt_sandbox.template.Template object - :param templatedir: local directory path in which to find template - :param imagedir: local directory in which to storage disk image - - Creates an instance private disk image, backed by the content - of a template. - """ - pass - - @abstractmethod - def get_env(self, template, templatedir): - """ - :param template: libvirt_sandbox.template.Template object - :param templatedir: local directory path in which to find template - - Get the dict of environment variables to set - """ - pass - - def post_run(self, template, templatedir, imagename): - """ - :param template: libvirt_sandbox.template.Template object - :param templatedir: local directory path in which to find template - :param imagename: name of the image that just stopped running - - Hook called after the image has been stopped. By default is doesn't - do anything, subclasses can override this to do some additional - cleanup. - """ - pass - - - # Utility functions to share between the sources. - - def format_disk(self,disk,format,connect): - cmd = ['virt-sandbox'] - if connect is not None: - cmd.append("-c") - cmd.append(connect) - cmd.append("-p") - params = ['--disk=file:disk_image=%s,format=%s' %(disk,format), - '/sbin/mkfs.ext3', - '/dev/disk/by-tag/disk_image'] - cmd = cmd + params - subprocess.check_call(cmd) - - def extract_tarball(self, diskfile, format, tarfile, connect): - cmd = ['virt-sandbox'] - if connect is not None: - cmd.append("-c") - cmd.append(connect) - cmd.append("-p") - compression = "" - if tarfile.endswith(".gz"): - compression = "z" - params = ['-m', - 'host-image:/mnt=%s,format=%s' % (diskfile, format), - '--', - '/bin/tar', - 'xf%s' % compression, - '%s' % tarfile, - '-C', - '/mnt'] - cmd = cmd + params - subprocess.check_call(cmd) diff --git a/libvirt-sandbox/image/sources/docker.py b/libvirt-sandbox/image/sources/docker.py deleted file mode 100755 index eaf41fc..0000000 --- a/libvirt-sandbox/image/sources/docker.py +++ /dev/null @@ -1,690 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2015 Universitat Politècnica de Catalunya. -# Copyright (C) 2015 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Author: Eren Yagdiran <erenyagdiran(a)gmail.com&gt; -# - -import sys -import json -import traceback -import os -import subprocess -import shutil -import urllib.error -import urllib.parse -import urllib.request -import hashlib -from abc import ABCMeta, abstractmethod -import copy -from libvirt_sandbox.image.template import Template - -from . import base - -class DockerConfParser(): - - def __init__(self,jsonfile): - with open(jsonfile) as json_file: - self.json_data = json.load(json_file) - def getCommand(self): - return self.json_data['config']['Cmd'] - def getEntrypoint(self): - return self.json_data['config']['Entrypoint'] - def getEnvs(self): - lst = self.json_data['config']['Env'] - if lst is not None and isinstance(lst,list): - return lst - else: - return [] - -class DockerImage(): - - def __init__(self, repo, name, tag=None): - - self.repo = repo - self.name = name - self.tag = tag - - if self.tag is None: - self.tag = "latest" - - if self.repo is None: - self.repo = "library" - - def __repr__(self): - return "%s/%s,tag=%s" % (self.repo, self.name, self.tag) - - @classmethod - def from_template(cls, template): - bits = template.path[1:].split("/") - if len(bits) == 1: - return cls(repo=None, - name=bits[0], - tag=template.params.get("tag")) - elif len(bits) == 2: - return cls(repo=bits[0], - name=bits[1], - tag=template.params.get("tag")) - else: - raise Exception("Expected image name, or repo & image name for path, not '%s'", - template.path) - - -class DockerAuth(): - - __metaclass__ = ABCMeta - def __init__(self): - pass - - @abstractmethod - def prepare_req(self, req): - pass - - @abstractmethod - def process_res(self, res): - pass - - @abstractmethod - def process_err(self, err): - return False - - -class DockerAuthNop(DockerAuth): - - def prepare_req(self, req): - pass - - def process_res(self, res): - pass - - def process_err(self, err): - return False - - -class DockerAuthBasic(DockerAuth): - - def __init__(self, username, password): - self.username = username - self.password = password - self.token = None - - def prepare_req(self, req): - if self.username is not None: - auth = base64.encodestring( - '%s:%s' % (self.username, self.password)).replace('\n', '') - - req.add_header("Authorization", "Basic %s" % auth) - - req.add_header("X-Docker-Token", "true") - - def process_res(self, res): - self.token = res.info().get('X-Docker-Token') - - def process_err(self, err): - return False - - -class DockerAuthToken(DockerAuth): - - def __init__(self, token): - self.token = token - - def prepare_req(self, req): - req.add_header("Authorization", "Token %s" % self.token) - - def process_res(self, res): - pass - - def process_err(self, err): - return False - - -class DockerAuthBearer(DockerAuth): - - def __init__(self): - self.token = None - - def prepare_req(self, req): - if self.token is not None: - req.add_header("Authorization", "Bearer %s" % self.token) - - def process_res(self, res): - pass - - def process_err(self, err): - method = err.headers.get("WWW-Authenticate", None) - if method is None: - return False - - if not method.startswith("Bearer "): - return False - - challenge = method[7:] - - bits = challenge.split(",") - attrs = {} - for bit in bits: - subbit = bit.split("=") - attrs[subbit[0]] = subbit[1][1:-1] - - url = attrs["realm"] - del attrs["realm"] - if "error" in attrs: - del attrs["error"] - - params = "&".join([ - "%s=%s" % (attr, attrs[attr]) - for attr in attrs.keys() - ]) - if params != "": - url = url + "?" + params - - req = urllib.request.Request(url=url) - req.add_header("Accept", "application/json") - - res = urllib.request.urlopen(req) - data = json.loads(res.read()) - self.token = data["token"] - return True - - -class DockerRegistry(): - - def __init__(self, uri_base): - - self.uri_base = list(urllib.parse.urlparse(uri_base)) - self.auth_handler = DockerAuthNop() - - def set_auth_handler(self, auth_handler): - self.auth_handler = auth_handler - - def supports_v2(self): - try: - (data, res) = self.get_json("/v2/") - ver = res.info().get("Docker-Distribution-Api-Version") - except urllib.error.HTTPError as e: - ver = e.headers.get("Docker-Distribution-Api-Version", None) - - if ver is None: - return False - return ver.startswith("registry/2") - - def set_server(self, server): - self.uri_base[1] = server - - @classmethod - def from_template(cls, template): - protocol = template.protocol - hostname = template.hostname - port = template.port - - if protocol is None: - protocol = "https" - if hostname is None: - hostname = "index.docker.io" - - if port is None: - server = hostname - else: - server = "%s:%s" % (hostname, port) - - url = urllib.parse.urlunparse((protocol, server, "", None, None, None)) - - return cls(url) - - def get_url(self, path, headers=None): - url_bits = copy.copy(self.uri_base) - url_bits[2] = path - url = urllib.parse.urlunparse(url_bits) - debug("Fetching %s..." % url) - - req = urllib.request.Request(url=url) - - if headers is not None: - for h in headers.keys(): - req.add_header(h, headers[h]) - - self.auth_handler.prepare_req(req) - - try: - res = urllib.request.urlopen(req) - self.auth_handler.process_res(res) - return res - except urllib.error.HTTPError as e: - if e.code == 401: - retry = self.auth_handler.process_err(e) - if retry: - debug("Re-Fetching %s..." % url) - self.auth_handler.prepare_req(req) - res = urllib.request.urlopen(req) - self.auth_handler.process_res(res) - return res - else: - debug("Not re-fetching") - raise - else: - raise - - def save_data(self, path, dest, checksum=None): - try: - res = self.get_url(path) - - datalen = res.info().get("Content-Length") - if datalen is not None: - datalen = int(datalen) - - csum = None - if checksum is not None: - csum = hashlib.sha256() - - pattern = [".", "o", "O", "o"] - patternIndex = 0 - donelen = 0 - - with open(dest, "wb") as f: - while 1: - buf = res.read(1024*64) - if not buf: - break - if csum is not None: - csum.update(buf) - f.write(buf) - - if datalen is not None: - donelen = donelen + len(buf) - debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % ( - pattern[patternIndex], (donelen/1024), (datalen/1024) - )) - patternIndex = (patternIndex + 1) % 4 - - debug("\x1b[K") - if csum is not None: - csumstr = "sha256:" + csum.hexdigest() - if csumstr != checksum: - debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum)) - os.remove(dest) - raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) - debug("OK\n") - return res - except Exception as e: - debug("FAIL %s\n" % str(e)) - raise - - def get_json(self, path): - try: - headers = {} - headers["Accept"] = "application/json" - res = self.get_url(path, headers) - data = json.loads(res.read()) - debug("OK\n") - return (data, res) - except Exception as e: - debug("FAIL %s\n" % str(e)) - raise - - -class DockerSource(base.Source): - - def _check_cert_validate(self): - major = sys.version_info.major - SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" - SSL_WARNING +="\nSee https://bugs.python.org/issue22417\n" - py2_7_9_hexversion = 34015728 - py3_4_3_hexversion = 50594800 - if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion): - sys.stderr.write(SSL_WARNING) - - def _was_downloaded(self, image, templatedir): - try: - self._get_image_list(image, templatedir) - return True - except Exception: - return False - - def list_templates(self, templatedir): - indexes = [] - try: - imagedirs = os.listdir(templatedir) - except OSError: - return [] - - for imagetagid in imagedirs: - indexfile = templatedir + "/" + imagetagid + "/index.json" - if os.path.exists(indexfile): - with open(indexfile,"r") as f: - index = json.load(f) - indexes.append(index) - - return [ - Template(source="docker", - protocol=None, - hostname=None, - port=None, - username=None, - password=None, - path="/%s/%s" % (index.get("repo", "library"), index["name"]), - params={ - "tag": index.get("tag", "latest"), - }) for index in indexes] - - def has_template(self, template, templatedir): - try: - image = DockerImage.from_template(template) - configfile, diskfile = self._get_template_data(image, templatedir) - return os.path.exists(diskfile) - except Exception: - return False - - def download_template(self, image, template, templatedir): - try: - createdFiles = [] - createdDirs = [] - - self._download_template_impl(image, template, templatedir, createdFiles, createdDirs) - except Exception as e: - for f in createdFiles: - try: - os.remove(f) - except: - pass - for d in createdDirs: - try: - shutil.rmtree(d) - except: - pass - raise - - def _download_template_impl(self, image, template, templatedir, createdFiles, createdDirs): - self._check_cert_validate() - - registry = DockerRegistry.from_template(template) - registry.set_auth_handler(DockerAuthBearer()) - if registry.supports_v2(): - self._download_template_impl_v2(registry, image, template, templatedir, createdFiles, createdDirs) - else: - self._download_template_impl_v1(registry, image, template, templatedir, createdFiles, createdDirs) - - def _download_template_impl_v1(self, registry, image, template, templatedir, createdFiles, createdDirs): - basicauth = DockerAuthBasic(template.username, template.password) - registry.set_auth_handler(basicauth) - try: - (data, res) = registry.get_json("/v1/repositories/%s/%s/images" % ( - image.repo, image.name, - )) - except urllib.error.HTTPError as e: - raise ValueError(["Image '%s' does not exist" % template]) - - registryendpoint = res.info().get('X-Docker-Endpoints') - - if basicauth.token is not None: - registry.set_auth_handler(DockerAuthToken(basicauth.token)) - else: - registry.set_auth_handler(DockerAuthNop()) - - (data, res) = registry.get_json("/v1/repositories/%s/%s/tags" %( - image.repo, image.name - )) - - if image.tag not in data: - raise ValueError(["Tag '%s' does not exist for image '%s'" % - (image.tag, template)]) - imagetagid = data[image.tag] - - (data, res) = registry.get_json("/v1/images/" + imagetagid + "/ancestry") - - if data[0] != imagetagid: - raise ValueError(["Expected first layer id '%s' to match image id '%s'", - data[0], imagetagid]) - - for layerid in data: - layerdir = templatedir + "/" + layerid - if not os.path.exists(layerdir): - os.makedirs(layerdir) - createdDirs.append(layerdir) - - jsonfile = layerdir + "/template.json" - datafile = layerdir + "/template.tar.gz" - - if not os.path.exists(jsonfile) or not os.path.exists(datafile): - res = registry.save_data("/v1/images/" + layerid + "/json", - jsonfile) - createdFiles.append(jsonfile) - - registry.save_data("/v1/images/" + layerid + "/layer", - datafile) - createdFiles.append(datafile) - - index = { - "repo": image.repo, - "name": image.name, - "tag": image.tag, - } - - indexfile = templatedir + "/" + imagetagid + "/index.json" - with open(indexfile, "w") as f: - f.write(json.dumps(index)) - - - def _download_template_impl_v2(self, registry, image, template, templatedir, createdFiles, createdDirs): - (manifest, res) = registry.get_json( "/v2/%s/%s/manifests/%s" % ( - image.repo, image.name, image.tag)) - - layerChecksums = [ - layer["blobSum"] for layer in manifest["fsLayers"] - ] - layers = [ - json.loads(entry["v1Compatibility"]) for entry in manifest["history"] - ] - - for i in range(len(layerChecksums)): - layerChecksum = layerChecksums[i] - config = layers[i] - - layerdir = templatedir + "/" + config["id"] - if not os.path.exists(layerdir): - os.makedirs(layerdir) - createdDirs.append(layerdir) - - jsonfile = layerdir + "/template.json" - datafile = layerdir + "/template.tar.gz" - - with open(jsonfile, "w") as fh: - fh.write(json.dumps(config)) - - registry.save_data("/v2/%s/%s/blobs/%s" % ( - image.repo, image.name, layerChecksum), - datafile, checksum=layerChecksum) - - - index = { - "repo": image.repo, - "name": image.name, - "tag": image.tag, - } - - indexfile = templatedir + "/" + layers[0]["id"] + "/index.json" - with open(indexfile, "w") as f: - f.write(json.dumps(index)) - - - def create_template(self, template, templatedir, connect=None): - image = DockerImage.from_template(template) - - if not self._was_downloaded(image, templatedir): - self.download_template(image, template, templatedir) - - imagelist = self._get_image_list(image, templatedir) - imagelist.reverse() - - parentImage = None - for imagetagid in imagelist: - templateImage = templatedir + "/" + imagetagid + "/template.qcow2" - cmd = ["qemu-img","create","-f","qcow2"] - if parentImage is not None: - cmd.append("-o") - cmd.append("backing_fmt=qcow2,backing_file=%s" % parentImage) - cmd.append(templateImage) - if parentImage is None: - cmd.append("10G") - subprocess.check_call(cmd) - - if parentImage is None: - self.format_disk(templateImage, "qcow2", connect) - - path = templatedir + "/" + imagetagid + "/template." - self.extract_tarball(path + "qcow2", - "qcow2", - path + "tar.gz", - connect) - parentImage = templateImage - - def _get_image_list(self, image, templatedir): - imageparent = {} - imagenames = {} - imagedirs = [] - try: - imagedirs = os.listdir(templatedir) - except OSError: - pass - for imagetagid in imagedirs: - indexfile = templatedir + "/" + imagetagid + "/index.json" - if os.path.exists(indexfile): - with open(indexfile,"r") as f: - index = json.load(f) - thisimage = DockerImage(index.get("repo"), - index["name"], - index.get("tag")) - imagenames[str(thisimage)] = imagetagid - jsonfile = templatedir + "/" + imagetagid + "/template.json" - if os.path.exists(jsonfile): - with open(jsonfile,"r") as f: - data = json.load(f) - parent = data.get("parent",None) - if parent: - imageparent[imagetagid] = parent - if str(image) not in imagenames: - raise ValueError(["Image %s does not exist locally" % image]) - imagetagid = imagenames[str(image)] - imagelist = [] - while imagetagid != None: - imagelist.append(imagetagid) - parent = imageparent.get(imagetagid,None) - imagetagid = parent - return imagelist - - def delete_template(self, template, templatedir): - image = DockerImage.from_template(template) - - imageusage = {} - imageparent = {} - imagenames = {} - imagedirs = [] - try: - imagedirs = os.listdir(templatedir) - except OSError: - pass - for imagetagid in imagedirs: - indexfile = templatedir + "/" + imagetagid + "/index.json" - if os.path.exists(indexfile): - with open(indexfile,"r") as f: - index = json.load(f) - thisimage = DockerImage(index.get("repo"), - index["name"], - index.get("tag")) - imagenames[str(thisimage)] = imagetagid - jsonfile = templatedir + "/" + imagetagid + "/template.json" - if os.path.exists(jsonfile): - with open(jsonfile,"r") as f: - data = json.load(f) - - parent = data.get("parent",None) - if parent: - if parent not in imageusage: - imageusage[parent] = [] - imageusage[parent].append(imagetagid) - imageparent[imagetagid] = parent - - - if str(image) not in imagenames: - raise ValueError(["Image %s does not exist locally" % image]) - imagetagid = imagenames[str(image)] - while imagetagid != None: - debug("Remove %s\n" % imagetagid) - parent = imageparent.get(imagetagid,None) - - indexfile = templatedir + "/" + imagetagid + "/index.json" - if os.path.exists(indexfile): - os.remove(indexfile) - jsonfile = templatedir + "/" + imagetagid + "/template.json" - if os.path.exists(jsonfile): - os.remove(jsonfile) - datafile = templatedir + "/" + imagetagid + "/template.tar.gz" - if os.path.exists(datafile): - os.remove(datafile) - imagedir = templatedir + "/" + imagetagid - shutil.rmtree(imagedir) - - if parent: - if len(imageusage[parent]) != 1: - debug("Parent %s is shared\n" % parent) - parent = None - imagetagid = parent - - def _get_template_data(self, image, templatedir): - imageList = self._get_image_list(image, templatedir) - toplayer = imageList[0] - diskfile = templatedir + "/" + toplayer + "/template.qcow2" - configfile = templatedir + "/" + toplayer + "/template.json" - return configfile, diskfile - - def get_disk(self, template, templatedir, imagedir, sandboxname): - image = DockerImage.from_template(template) - configfile, diskfile = self._get_template_data(image, templatedir) - tempfile = imagedir + "/" + sandboxname.split('/')[-1] + ".qcow2" - if not os.path.exists(imagedir): - os.makedirs(imagedir) - cmd = ["qemu-img","create","-q","-f","qcow2"] - cmd.append("-o") - cmd.append("backing_fmt=qcow2,backing_file=%s" % diskfile) - cmd.append(tempfile) - subprocess.check_call(cmd) - return tempfile - - def get_command(self, template, templatedir, userargs): - image = DockerImage.from_template(template) - configfile, diskfile = self._get_template_data(image, templatedir) - configParser = DockerConfParser(configfile) - cmd = configParser.getCommand() - entrypoint = configParser.getEntrypoint() - if entrypoint is None: - entrypoint = [] - if cmd is None: - cmd = [] - if userargs is not None and len(userargs) > 0: - return entrypoint + userargs - else: - return entrypoint + cmd - - def get_env(self, template, templatedir): - image = DockerImage.from_template(template) - configfile, diskfile = self._get_template_data(image, templatedir) - configParser = DockerConfParser(configfile) - return configParser.getEnvs() - -def debug(msg): - sys.stderr.write(msg) diff --git a/libvirt-sandbox/image/sources/virtbuilder.py b/libvirt-sandbox/image/sources/virtbuilder.py deleted file mode 100755 index 1b32083..0000000 --- a/libvirt-sandbox/image/sources/virtbuilder.py +++ /dev/null @@ -1,109 +0,0 @@ -# -# Copyright (C) 2015 SUSE LLC -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -# -# Author: Cedric Bosdonnat <cbosdonnat(a)suse.com&gt; -# - -import os -import os.path -import subprocess - -from . import base -from libvirt_sandbox.image.template import Template - - -class VirtBuilderSource(base.Source): - - def _get_template_name(self, template): - # We shouldn't have '/' in the names, but let's make sure - # nobody can try to alter the folders structure later. - return template.path[1:].replace('/', '_') - - def has_template(self, template, templatedir): - imagepath = "%s/%s.qcow2" % (templatedir, template.path) - return os.path.exists(imagepath) - - def create_template(self, template, templatedir, connect=None): - if not os.path.exists(templatedir): - os.makedirs(templatedir) - - # Get the image using virt-builder - templatename = self._get_template_name(template) - imagepath_original = "%s/%s-original.qcow2" % (templatedir, templatename) - imagepath = "%s/%s.qcow2" % (templatedir, templatename) - cmd = ["virt-builder", templatename, "--no-network", - "-o", imagepath_original, "--format", "qcow2"] - subprocess.check_call(cmd) - - try: - # We need to convert this image into a single partition one. - tarfile = "%s/%s.tar" % (templatedir, templatename) - cmd = ["virt-tar-out", "-a", imagepath_original, "/", tarfile] - subprocess.check_call(cmd) - - cmd = ["qemu-img", "create", "-q", "-f", "qcow2", imagepath, "10G"] - subprocess.check_call(cmd) - - self.format_disk(imagepath, "qcow2", connect) - self.extract_tarball(imagepath, "qcow2", tarfile, connect) - - finally: - os.unlink(imagepath_original) - os.unlink(tarfile) - - def list_templates(self, templatedir): - files = [] - try: - imagefiles = os.listdir(templatedir) - except OSError: - return [] - - for filename in imagefiles: - if not filename.endswith(".qcow2"): - continue - files.append(filename[0:-6]) - - return [ - Template(source="virt-builder", - protocol=None, - hostname=None, - port=None, - username=None, - password=None, - path="/%s" % filename, - params={}) for filename in files] - - def delete_template(self, template, templatedir): - os.unlink("%s/%s.qcow2" % (templatedir, self._get_template_name(template))) - - def get_command(self, template, templatedir, userargs): - return userargs - - def get_disk(self,template, templatedir, imagedir, sandboxname): - diskfile = "%s/%s.qcow2" % (templatedir, self._get_template_name(template)) - tempfile = imagedir + "/" + sandboxname + ".qcow2" - if not os.path.exists(imagedir): - os.makedirs(imagedir) - cmd = ["qemu-img", "create", "-q", - "-f", "qcow2", - "-o", "backing_fmt=qcow2,backing_file=%s" % diskfile, - tempfile] - subprocess.check_call(cmd) - return tempfile - - def get_env(self,template, templatedir): - return [] diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py deleted file mode 100644 index ab2ea29..0000000 --- a/libvirt-sandbox/image/template.py +++ /dev/null @@ -1,133 +0,0 @@ -# -# -*- coding: utf-8 -*- -# Authors: Daniel P. Berrange <berrange(a)redhat.com&gt; -# -# Copyright (C) 2015 Red Hat, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -# - -import urllib.parse -import importlib -import re - -class Template(object): - - def __init__(self, - source, protocol, - hostname, port, - username, password, - path, params): - """ - :param source: template source name - :param protocol: network transport protocol or None - :param hostname: registry hostname or None - :param port: registry port or None - :param username: username or None - :param password: password or None - :param path: template path identifier - :param params: template parameters - - docker:///ubuntu - - docker+https://index.docker.io/ubuntu?tag=latest - - virt-builder:///fedora-20 - """ - - self.source = source - self.protocol = protocol - self.hostname = hostname - self.port = port - self.username = username - self.password = password - self.path = path - self.params = params - if self.params is None: - self.params = {} - - @classmethod - def _get_source_impl(klass, source): - try: - p = re.compile("\W") - sourcemod = "".join(p.split(source)) - sourcename = "".join([i.capitalize() for i in p.split(source)]) - - mod = importlib.import_module( - "libvirt_sandbox.image.sources." + sourcemod) - classname = sourcename + "Source" - classimpl = getattr(mod, classname) - return classimpl() - except Exception as e: - print (e) - raise Exception("Invalid source: '%s'" % source) - - def get_source_impl(self): - if self.source == "": - raise Exception("Missing scheme in image URI") - - return self._get_source_impl(self.source) - - def __repr__(self): - if self.protocol is not None: - scheme = self.source + "+" + self.protocol - else: - scheme = self.source - if self.hostname: - if self.port: - netloc = "%s:%d" % (self.hostname, self.port) - else: - netloc = self.hostname - - if self.username: - if self.password: - auth = self.username + ":" + self.password - else: - auth = self.username - netloc = auth + "@" + netloc - else: - netloc = None - - query = "&".join([key + "=" + self.params[key] for key in self.params.keys()]) - ret = urllib.parse.urlunparse((scheme, netloc, self.path, None, query, None)) - return ret - - @classmethod - def from_uri(klass, uri): - o = urllib.parse.urlparse(uri) - - idx = o.scheme.find("+") - if idx == -1: - source = o.scheme - protocol = None - else: - source = o.scheme[0:idx] - protocol = o.scheme[idx + 1:] - - query = {} - if o.query is not None and o.query != "": - for param in o.query.split("&"): - (key, val) = param.split("=") - query[key] = val - return klass(source, protocol, - o.hostname, o.port, - o.username, o.password, - o.path, query) - - @classmethod - def get_all(klass, source, templatedir): - impl = klass._get_source_impl(source) - - return impl.list_templates(templatedir) diff --git a/m4/virt-win32.m4 b/m4/virt-win32.m4 index d088ce8..8b908e7 100644 --- a/m4/virt-win32.m4 +++ b/m4/virt-win32.m4 @@ -4,15 +4,11 @@ AC_DEFUN([LIBVIRT_SANDBOX_WIN32],[ dnl for now since I'm not supporting mingw at present. - RWMJ CYGWIN_EXTRA_LDFLAGS= CYGWIN_EXTRA_LIBADD= - CYGWIN_EXTRA_PYTHON_LIBADD= MINGW_EXTRA_LDFLAGS= case "$host" in *-*-cygwin*) CYGWIN_EXTRA_LDFLAGS="-no-undefined" CYGWIN_EXTRA_LIBADD="${INTLLIBS}" - if test "x$PYTHON_VERSION" != "x"; then - CYGWIN_EXTRA_PYTHON_LIBADD="-L/usr/lib/python${PYTHON_VERSION}/config -lpython${PYTHON_VERSION}" - fi ;; *-*-mingw*) MINGW_EXTRA_LDFLAGS="-no-undefined" @@ -20,6 +16,5 @@ AC_DEFUN([LIBVIRT_SANDBOX_WIN32],[ esac AC_SUBST([CYGWIN_EXTRA_LDFLAGS]) AC_SUBST([CYGWIN_EXTRA_LIBADD]) - AC_SUBST([CYGWIN_EXTRA_PYTHON_LIBADD]) AC_SUBST([MINGW_EXTRA_LDFLAGS]) ])