- Devel - libvirt List Archives

[libvirt] [PATCH v2 sandbox] virt-sandbox-image: switch to use URI to identify templates
by Daniel P. Berrange 22 Sep '15

22 Sep '15

Currently the CLI syntax is somewhat docker specific requiring inclusion of --registry arg to identify the docker download server. Other app containers have a notion of download server, but don't separate it from the template name. This patch removes that docker-ism by changing to use a URI for identifying the template image. So instead of virt-sandbox-image download \ --source docker --registry index.docker.io --username dan --password 123456 ubuntu:15.04 You can use virt-sandbox-image download docker://dan:123456@index.docker.io/ubuntu?tag=15.04 The only mandatory part is the source prefix and image name, so that can shorten to just virt-sandbox-image download docker:///ubuntu to pull down the latest ubuntu image, from the default registry using no authentication. --- Changed in v2: - Rebase against master, instead of (unpushed) docker volume patch libvirt-sandbox/image/cli.py | 71 +++++-------- libvirt-sandbox/image/sources/DockerSource.py | 142 ++++++++++++++------------ libvirt-sandbox/image/sources/Source.py | 29 +++--- libvirt-sandbox/image/template.py | 110 ++++++++++++++++++++ 4 files changed, 228 insertions(+), 124 deletions(-) create mode 100644 libvirt-sandbox/image/template.py diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py index 1718cc5..4d02289 100755 --- a/libvirt-sandbox/image/cli.py +++ b/libvirt-sandbox/image/cli.py @@ -3,7 +3,7 @@ # Authors: Daniel P. Berrange <berrange(a)redhat.com> # Eren Yagdiran <erenyagdiran(a)gmail.com> # -# Copyright (C) 2013 Red Hat, Inc. +# Copyright (C) 2013-2015 Red Hat, Inc. # Copyright (C) 2015 Universitat Politècnica de Catalunya. # # This program is free software; you can redistribute it and/or modify @@ -34,6 +34,8 @@ import subprocess import random import string +from libvirt_sandbox.image import template + if os.geteuid() == 0: default_template_dir = "/var/lib/libvirt/templates" default_image_dir = "/var/lib/libvirt/images" @@ -44,15 +46,6 @@ else: debug = False verbose = False -import importlib -def dynamic_source_loader(name): - name = name[0].upper() + name[1:] - modname = "libvirt_sandbox.image.sources." + name + "Source" - mod = importlib.import_module(modname) - classname = name + "Source" - classimpl = getattr(mod, classname) - return classimpl() - gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") gettext.textdomain("libvirt-sandbox") try: @@ -73,11 +66,10 @@ def info(msg): def download(args): try: - dynamic_source_loader(args.source).download_template(templatename=args.template, - templatedir=args.template_dir, - registry=args.registry, - username=args.username, - password=args.password) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.download_template(template=tmpl, + templatedir=args.template_dir) except IOError,e: print "Source %s cannot be found in given path" %args.source except Exception,e: @@ -85,17 +77,21 @@ def download(args): def delete(args): try: - dynamic_source_loader(args.source).delete_template(templatename=args.template, - templatedir=args.template_dir) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.delete_template(template=tmpl, + templatedir=args.template_dir) except Exception,e: print "Delete Error %s", str(e) def create(args): try: - dynamic_source_loader(args.source).create_template(templatename=args.template, - templatedir=args.template_dir, - connect=args.connect, - format=args.format) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.create_template(template=tmpl, + templatedir=args.template_dir, + connect=args.connect, + format=args.format) except Exception,e: print "Create Error %s" % str(e) @@ -103,19 +99,22 @@ def run(args): try: if args.connect is not None: check_connect(args.connect) - source = dynamic_source_loader(args.source) + + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + name = args.name if name is None: randomid = ''.join(random.choice(string.lowercase) for i in range(10)) - name = args.template + ":" + randomid + name = tmpl.path[1:] + ":" + randomid - diskfile = source.get_disk(templatename=args.template, + diskfile = source.get_disk(template=tmpl, templatedir=args.template_dir, imagedir=args.image_dir, sandboxname=name) format = "qcow2" - commandToRun = source.get_command(args.template, args.template_dir, args.args) + commandToRun = source.get_command(tmpl, args.template_dir, args.args) if len(commandToRun) == 0: commandToRun = ["/bin/sh"] cmd = ['virt-sandbox', '--name', name] @@ -129,7 +128,7 @@ def run(args): params.append('-N') params.append(networkArgs) - allEnvs = source.get_env(args.template, args.template_dir) + allEnvs = source.get_env(tmpl, args.template_dir) envArgs = args.env if envArgs is not None: allEnvs = allEnvs + envArgs @@ -151,7 +150,7 @@ def run(args): def requires_template(parser): parser.add_argument("template", - help=_("name of the template")) + help=_("URI of the template")) def requires_name(parser): parser.add_argument("-n","--name", @@ -163,23 +162,10 @@ def check_connect(connectstr): raise ValueError("URI '%s' is not supported by virt-sandbox-image" % connectstr) return True -def requires_source(parser): - parser.add_argument("-s","--source", - default="docker", - help=_("name of the template")) - def requires_connect(parser): parser.add_argument("-c","--connect", help=_("Connect string for libvirt")) -def requires_auth_conn(parser): - parser.add_argument("-r","--registry", - help=_("Url of the custom registry")) - parser.add_argument("-u","--username", - help=_("Username for the custom registry")) - parser.add_argument("-p","--password", - help=_("Password for the custom registry")) - def requires_template_dir(parser): global default_template_dir parser.add_argument("-t","--template-dir", @@ -196,8 +182,6 @@ def gen_download_args(subparser): parser = subparser.add_parser("download", help=_("Download template data")) requires_template(parser) - requires_source(parser) - requires_auth_conn(parser) requires_template_dir(parser) parser.set_defaults(func=download) @@ -205,7 +189,6 @@ def gen_delete_args(subparser): parser = subparser.add_parser("delete", help=_("Delete template data")) requires_template(parser) - requires_source(parser) requires_template_dir(parser) parser.set_defaults(func=delete) @@ -213,7 +196,6 @@ def gen_create_args(subparser): parser = subparser.add_parser("create", help=_("Create image from template data")) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) parser.add_argument("-f","--format", @@ -226,7 +208,6 @@ def gen_run_args(subparser): help=_("Run an already built image")) requires_name(parser) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) requires_image_dir(parser) diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py index c374a0c..10f8537 100644 --- a/libvirt-sandbox/image/sources/DockerSource.py +++ b/libvirt-sandbox/image/sources/DockerSource.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -28,6 +29,8 @@ import traceback import os import subprocess import shutil +import urlparse + class DockerConfParser(): @@ -47,12 +50,6 @@ class DockerConfParser(): class DockerSource(Source): - www_auth_username = None - www_auth_password = None - - def __init__(self): - self.default_index_server = "index.docker.io" - def _check_cert_validate(self): major = sys.version_info.major SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" @@ -62,43 +59,38 @@ class DockerSource(Source): if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion): sys.stderr.write(SSL_WARNING) - def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): - if registry is None: - registry = self.default_index_server - - if username is not None: - self.www_auth_username = username - self.www_auth_password = password - + def download_template(self, template, templatedir): self._check_cert_validate() - tag = "latest" - offset = templatename.find(':') - if offset != -1: - tag = templatename[offset + 1:] - templatename = templatename[0:offset] + try: - (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images", - {"X-Docker-Token": "true"}) + (data, res) = self._get_json(template, + None, + "/v1/repositories" + template.path + "/images", + {"X-Docker-Token": "true"}) except urllib2.HTTPError, e: - raise ValueError(["Image '%s' does not exist" % templatename]) + raise ValueError(["Image '%s' does not exist" % template]) registryendpoint = res.info().getheader('X-Docker-Endpoints') token = res.info().getheader('X-Docker-Token') checksums = {} for layer in data: pass - (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags", - { "Authorization": "Token " + token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/repositories" + template.path + "/tags", + { "Authorization": "Token " + token }) cookie = res.info().getheader('Set-Cookie') + tag = template.params.get("tag", "latest") if not tag in data: - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)]) + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, template)]) imagetagid = data[tag] - (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry", - { "Authorization": "Token "+token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/images/" + imagetagid + "/ancestry", + { "Authorization": "Token "+ token }) if data[0] != imagetagid: raise ValueError(["Expected first layer id '%s' to match image id '%s'", @@ -118,8 +110,11 @@ class DockerSource(Source): datafile = layerdir + "/template.tar.gz" if not os.path.exists(jsonfile) or not os.path.exists(datafile): - res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json", - { "Authorization": "Token " + token }, jsonfile) + res = self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/json", + { "Authorization": "Token " + token }, + jsonfile) createdFiles.append(jsonfile) layersize = int(res.info().getheader("Content-Length")) @@ -128,12 +123,15 @@ class DockerSource(Source): if layerid in checksums: datacsum = checksums[layerid] - self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer", - { "Authorization": "Token "+token }, datafile, datacsum, layersize) + self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/layer", + { "Authorization": "Token "+token }, + datafile, datacsum, layersize) createdFiles.append(datafile) index = { - "name": templatename, + "name": template.path, } indexfile = templatedir + "/" + imagetagid + "/index.json" @@ -152,9 +150,11 @@ class DockerSource(Source): shutil.rmtree(d) except: pass - def _save_data(self,server, path, headers, dest, checksum=None, datalen=None): + + def _save_data(self, template, server, path, headers, + dest, checksum=None, datalen=None): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers) csum = None if checksum is not None: @@ -193,8 +193,22 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise - def _get_url(self,server, path, headers): - url = "https://" + server + path + def _get_url(self, template, server, path, headers): + if template.protocol is None: + protocol = "https" + else: + protocol = template.protocol + + if server is None: + if template.hostname is None: + server = "index.docker.io" + else: + if template.port is not None: + server = template.hostname + ":" + template.port + else: + server = template.hostname + + url = urlparse.urlunparse((protocol, server, path, None, None, None)) debug("Fetching %s..." % url) req = urllib2.Request(url=url) @@ -204,16 +218,18 @@ class DockerSource(Source): req.add_header(h, headers[h]) #www Auth header starts - if self.www_auth_username is not None: - base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '') + if template.username and template.password: + base64string = base64.encodestring( + '%s:%s' % (template.username, + template.password)).replace('\n', '') req.add_header("Authorization", "Basic %s" % base64string) #www Auth header finish return urllib2.urlopen(req) - def _get_json(self,server, path, headers): + def _get_json(self, template, server, path, headers): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers) data = json.loads(res.read()) debug("OK\n") return (data, res) @@ -221,11 +237,11 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise - def create_template(self, templatename, templatedir, connect=None, format=None): + def create_template(self, template, templatedir, connect=None, format=None): if format is None: format = self.default_disk_format self._check_disk_format(format) - imagelist = self._get_image_list(templatename,templatedir) + imagelist = self._get_image_list(template, templatedir) imagelist.reverse() parentImage = None @@ -252,7 +268,7 @@ class DockerSource(Source): if not format in supportedFormats: raise ValueError(["Unsupported image format %s" % format]) - def _get_image_list(self,templatename,destdir): + def _get_image_list(self, template, destdir): imageparent = {} imagenames = {} imagedirs = os.listdir(destdir) @@ -265,13 +281,13 @@ class DockerSource(Source): jsonfile = destdir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) - parent = template.get("parent",None) + data = json.load(f) + parent = data.get("parent",None) if parent: imageparent[imagetagid] = parent - if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) - imagetagid = imagenames[templatename] + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path]) + imagetagid = imagenames[template.path] imagelist = [] while imagetagid != None: imagelist.append(imagetagid) @@ -310,7 +326,7 @@ class DockerSource(Source): cmd = cmd + params subprocess.call(cmd) - def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): imageusage = {} imageparent = {} imagenames = {} @@ -324,9 +340,9 @@ class DockerSource(Source): jsonfile = templatedir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) + data = json.load(f) - parent = template.get("parent",None) + parent = data.get("parent",None) if parent: if parent not in imageusage: imageusage[parent] = [] @@ -334,10 +350,10 @@ class DockerSource(Source): imageparent[imagetagid] = parent - if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path]) - imagetagid = imagenames[templatename] + imagetagid = imagenames[template.path] while imagetagid != None: debug("Remove %s\n" % imagetagid) parent = imageparent.get(imagetagid,None) @@ -360,15 +376,15 @@ class DockerSource(Source): parent = None imagetagid = parent - def _get_template_data(self, templatename, templatedir): - imageList = self._get_image_list(templatename, templatedir) + def _get_template_data(self, template, templatedir): + imageList = self._get_image_list(template, templatedir) toplayer = imageList[0] diskfile = templatedir + "/" + toplayer + "/template.qcow2" configfile = templatedir + "/" + toplayer + "/template.json" return configfile, diskfile - def get_disk(self,templatename, templatedir, imagedir, sandboxname): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_disk(self, template, templatedir, imagedir, sandboxname): + configfile, diskfile = self._get_template_data(template, templatedir) tempfile = imagedir + "/" + sandboxname + ".qcow2" if not os.path.exists(imagedir): os.makedirs(imagedir) @@ -379,8 +395,8 @@ class DockerSource(Source): subprocess.call(cmd) return tempfile - def get_command(self, templatename, templatedir, userargs): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_command(self, template, templatedir, userargs): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) cmd = configParser.getCommand() entrypoint = configParser.getEntrypoint() @@ -393,8 +409,8 @@ class DockerSource(Source): else: return entrypoint + cmd - def get_env(self, templatename, templatedir): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_env(self, template, templatedir): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) return configParser.getEnvs() diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py index 8a21f90..597a7fb 100644 --- a/libvirt-sandbox/image/sources/Source.py +++ b/libvirt-sandbox/image/sources/Source.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -33,14 +34,10 @@ class Source(): pass @abstractmethod - def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): + def download_template(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template - :param registry: optional hostname of image registry server - :param username: optional username to authenticate against registry server - :param password: optional password to authenticate against registry server Download a template from the registry, storing it in the local filesystem @@ -48,10 +45,10 @@ class Source(): pass @abstractmethod - def create_template(self, templatename, templatedir, + def create_template(self, template, templatedir, connect=None, format=None): """ - :param templatename: name of the template image to create + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template :param connect: libvirt connection URI :param format: disk image format @@ -63,9 +60,9 @@ class Source(): pass @abstractmethod - def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): """ - :param templatename: name of the template image to delete + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path from which to delete template Delete all local files associated with the template @@ -73,9 +70,9 @@ class Source(): pass @abstractmethod - def get_command(self, templatename, templatedir, userargs): + def get_command(self, template, templatedir, userargs): """ - :param templatename: name of the template image to query + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which templates are stored :param userargs: user specified arguments to run @@ -85,9 +82,9 @@ class Source(): pass @abstractmethod - def get_disk(self,templatename, templatedir, imagedir, sandboxname): + def get_disk(self, template, templatedir, imagedir, sandboxname): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template :param imagedir: local directory in which to storage disk image @@ -97,9 +94,9 @@ class Source(): pass @abstractmethod - def get_env(self,templatename, templatedir): + def get_env(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template Get the dict of environment variables to set diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py new file mode 100644 index 0000000..0ad767b --- /dev/null +++ b/libvirt-sandbox/image/template.py @@ -0,0 +1,110 @@ +# +# -*- coding: utf-8 -*- +# Authors: Daniel P. Berrange <berrange(a)redhat.com> +# +# Copyright (C) 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# + +import urlparse +import importlib + +class Template(object): + + def __init__(self, + source, protocol, + hostname, port, + username, password, + path, params): + """ + :param source: template source name + :param protocol: network transport protocol or None + :param hostname: registry hostname or None + :param port: registry port or None + :param username: username or None + :param password: password or None + :param path: template path identifier + :param params: template parameters + + docker:///ubuntu + + docker+https://index.docker.io/ubuntu?tag=latest + """ + + self.source = source + self.protocol = protocol + self.hostname = hostname + self.port = port + self.username = username + self.password = password + self.path = path + self.params = params + if self.params is None: + self.params = {} + + def get_source_impl(self): + mod = importlib.import_module( + "libvirt_sandbox.image.sources." + + self.source.capitalize() + "Source") + classname = self.source.capitalize() + "Source" + classimpl = getattr(mod, classname) + return classimpl() + + def __repr__(self): + if self.protocol is not None: + scheme = self.source + "+" + self.protocol + else: + scheme = self.source + if self.hostname: + if self.port: + netloc = self.hostname + ":" + self.port + else: + netloc = self.hostname + + if self.username: + if self.password: + auth = self.username + ":" + self.password + else: + auth = self.username + netloc = auth + "@" + netloc + else: + netloc = None + + query = "&".join([key + "=" + self.params[key] for key in self.params.keys()]) + return urlparse.urlunparse((scheme, netloc, self.path, None, query, None)) + + @classmethod + def from_uri(klass, uri): + o = urlparse.urlparse(uri) + + idx = o.scheme.find("+") + if idx == -1: + source = o.scheme + protocol = None + else: + source = o.scheme[0:idx] + protocol = o.schema[idx + 1:] + + query = {} + if o.query is not None and o.query != "": + for param in o.query.split("&"): + (key, val) = param.split("=") + query[key] = val + return klass(source, protocol, + o.hostname, o.port, + o.username, o.password, + o.path, query) + -- 2.4.3

2 8

[libvirt] [PATCH sandbox] docker: don't assume X-Docker-Token is set
by Daniel P. Berrange 22 Sep '15

22 Sep '15

The Red Hat docker registry (registry.access.redhat.com) does not set any X-Docker-Token HTTP header in its responses. Change the code so it only passes around this header if it is actually present. Signed-off-by: Daniel P. Berrange <berrange(a)redhat.com> --- libvirt-sandbox/image/sources/DockerSource.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py index f367c8f..78b2a53 100644 --- a/libvirt-sandbox/image/sources/DockerSource.py +++ b/libvirt-sandbox/image/sources/DockerSource.py @@ -83,10 +83,14 @@ class DockerSource(Source): checksums = {} for layer in data: pass + + headers = {} + if token is not None: + headers["Authorization"] = "Token" + token (data, res) = self._get_json(template, registryendpoint, "/v1/repositories" + template.path + "/tags", - { "Authorization": "Token " + token }) + headers) cookie = res.info().getheader('Set-Cookie') @@ -98,7 +102,7 @@ class DockerSource(Source): (data, res) = self._get_json(template, registryendpoint, "/v1/images/" + imagetagid + "/ancestry", - { "Authorization": "Token "+ token }) + headers) if data[0] != imagetagid: raise ValueError(["Expected first layer id '%s' to match image id '%s'", @@ -121,7 +125,7 @@ class DockerSource(Source): res = self._save_data(template, registryendpoint, "/v1/images/" + layerid + "/json", - { "Authorization": "Token " + token }, + headers, jsonfile) createdFiles.append(jsonfile) @@ -134,7 +138,7 @@ class DockerSource(Source): self._save_data(template, registryendpoint, "/v1/images/" + layerid + "/layer", - { "Authorization": "Token "+token }, + headers, datafile, datacsum, layersize) createdFiles.append(datafile) -- 2.4.3

2 4

[libvirt] [libvirt-python][PATCH] generator: fix build fail with old xml lib
by Luyao Huang 22 Sep '15

22 Sep '15

https://bugzilla.redhat.com/show_bug.cgi?id=1222795#c6 if build libvirt-python with some old xml lib (python-pyxml), build will fail and error like this: File "generator.py", line 139, in start if "string" in attrs: File "/usr/local/lib/python2.7/site-packages/_xmlplus/sax/xmlreader.py" \ , line 316, in __getitem__ return self._attrs[name] KeyError: 0 This is an old issue and have been mentioned in commit 3ae0a76d. There is no __contains__ in class AttributesImpl, python will use __getitem__ in this place, so we will get error. Let's use 'YYY in XXX.keys()' to avoid this issue. Signed-off-by: Luyao Huang <lhuang(a)redhat.com> --- generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator.py b/generator.py index 2fc838c..d9ae17e 100755 --- a/generator.py +++ b/generator.py @@ -136,7 +136,7 @@ class docParser(xml.sax.handler.ContentHandler): elif attrs['file'] == "libvirt-qemu": qemu_enum(attrs['type'],attrs['name'],attrs['value']) elif tag == "macro": - if "string" in attrs: + if "string" in attrs.keys(): params.append((attrs['name'], attrs['string'])) def end(self, tag): -- 1.8.3.1

3 2

[libvirt] [PATCH 0/4] Implement mockup capabilities cache in QEMU tests
by Ján Tomko 22 Sep '15

22 Sep '15

v2: https://www.redhat.com/archives/libvir-list/2015-September/msg00374.html v3: * split out the movement of existing initialization code to qemuTestDriverInit from the cache addition * qemuTestDriverInit now adds an empty capability cache, so most tests are unchanged * renamed 'cleanup' to 'error' in qemuTestDriverInit, since it's only called on error paths * fixed indentation issues to pass syntax-check * removed mode changes to 755 Pavel Fedin (4): tests: split out common qemu driver initialization Implement infrastracture for mocking up QEMU capabilities cache Use mockup cache Removed unneeded check src/qemu/qemu_capabilities.c | 16 ++++----- src/qemu/qemu_capspriv.h | 36 ++++++++++++++++++++ src/qemu/qemu_domain.c | 10 ++---- tests/domainsnapshotxml2xmltest.c | 10 ++---- tests/qemuagenttest.c | 11 +++--- tests/qemuargv2xmltest.c | 12 ++----- tests/qemuhotplugtest.c | 32 +++++++++--------- tests/qemuxml2argvtest.c | 17 +++++----- tests/qemuxml2xmltest.c | 8 ++--- tests/qemuxmlnstest.c | 17 +++++----- tests/testutilsqemu.c | 70 +++++++++++++++++++++++++++++++++++++++ tests/testutilsqemu.h | 7 ++++ 12 files changed, 170 insertions(+), 76 deletions(-) create mode 100644 src/qemu/qemu_capspriv.h -- 2.4.6

3 14

[libvirt] [PATCH] test driver: don't unlock pool after freeing it
by David Mansfield 21 Sep '15

21 Sep '15

The attached patch (taken from my modified Fedora 22 source rpm, 1.2.13.1-2.fc22, sorry), fixes a case where, in the test driver, memory is accessed after it's freed. Patch applies to latest git with: Hunk #1 succeeded at 4395 (offset -469 lines). The illegal access was found using valgrind. The function testStoragePoolUndefine() calls virStoragePoolObjRemove() (which seems to free the memory for that pool structure) and then in the cleanup stanza calls virStoragePoolObjUnlock() which tampers with the freed structure. -- Thanks, David Mansfield

2 3

[libvirt] [PATCH sandbox] virt-sandbox-image: switch to use URI to identify templates
by Daniel P. Berrange 21 Sep '15

21 Sep '15

Currently the CLI syntax is somewhat docker specific requiring inclusion of --registry arg to identify the docker download server. Other app containers have a notion of download server, but don't separate it from the template name. This patch removes that docker-ism by changing to use a URI for identifying the template image. So instead of virt-sandbox-image download \ --source docker --registry index.docker.io --username dan --password 123456 ubuntu:15.04 You can use virt-sandbox-image download docker://dan@123456:index.docker.io/ubuntu?tag=15.04 The only mandatory part is the source prefix and image name, so that can shorten to just virt-sandbox-image download docker:///ubuntu to pull down the latest ubuntu image, from the default registry using no authentication. --- libvirt-sandbox/image/cli.py | 73 +++++-------- libvirt-sandbox/image/sources/DockerSource.py | 146 ++++++++++++++------------ libvirt-sandbox/image/sources/Source.py | 33 +++--- libvirt-sandbox/image/template.py | 109 +++++++++++++++++++ 4 files changed, 232 insertions(+), 129 deletions(-) create mode 100644 libvirt-sandbox/image/template.py diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py index c8b5ba5..dd109a8 100755 --- a/libvirt-sandbox/image/cli.py +++ b/libvirt-sandbox/image/cli.py @@ -3,7 +3,7 @@ # Authors: Daniel P. Berrange <berrange(a)redhat.com> # Eren Yagdiran <erenyagdiran(a)gmail.com> # -# Copyright (C) 2013 Red Hat, Inc. +# Copyright (C) 2013-2015 Red Hat, Inc. # Copyright (C) 2015 Universitat Politècnica de Catalunya. # # This program is free software; you can redistribute it and/or modify @@ -34,6 +34,8 @@ import subprocess import random import string +from libvirt_sandbox.image import template + if os.geteuid() == 0: default_template_dir = "/var/lib/libvirt/templates" default_image_dir = "/var/lib/libvirt/images" @@ -44,15 +46,6 @@ else: debug = False verbose = False -import importlib -def dynamic_source_loader(name): - name = name[0].upper() + name[1:] - modname = "libvirt_sandbox.image.sources." + name + "Source" - mod = importlib.import_module(modname) - classname = name + "Source" - classimpl = getattr(mod, classname) - return classimpl() - gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale") gettext.textdomain("libvirt-sandbox") try: @@ -73,11 +66,10 @@ def info(msg): def download(args): try: - dynamic_source_loader(args.source).download_template(templatename=args.template, - templatedir=args.template_dir, - registry=args.registry, - username=args.username, - password=args.password) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.download_template(template=tmpl, + templatedir=args.template_dir) except IOError,e: print "Source %s cannot be found in given path" %args.source except Exception,e: @@ -85,17 +77,21 @@ def download(args): def delete(args): try: - dynamic_source_loader(args.source).delete_template(templatename=args.template, - templatedir=args.template_dir) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.delete_template(template=tmpl, + templatedir=args.template_dir) except Exception,e: print "Delete Error %s", str(e) def create(args): try: - dynamic_source_loader(args.source).create_template(templatename=args.template, - templatedir=args.template_dir, - connect=args.connect, - format=args.format) + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + source.create_template(template=tmpl, + templatedir=args.template_dir, + connect=args.connect, + format=args.format) except Exception,e: print "Create Error %s" % str(e) @@ -104,19 +100,22 @@ def run(args): global image_dir if args.connect is not None: check_connect(args.connect) - source = dynamic_source_loader(args.source) + + tmpl = template.Template.from_uri(args.template) + source = tmpl.get_source_impl() + name = args.name if name is None: randomid = ''.join(random.choice(string.lowercase) for i in range(10)) - name = args.template + ":" + randomid + name = tmpl.path[1:] + ":" + randomid - diskfile = source.get_disk(templatename=args.template, + diskfile = source.get_disk(template=tmpl, templatedir=args.template_dir, imagedir=args.image_dir, sandboxname=name) format = "qcow2" - commandToRun = source.get_command(args.template, args.template_dir, args.args) + commandToRun = source.get_command(tmpl, args.template_dir, args.args) if len(commandToRun) == 0: commandToRun = ["/bin/sh"] cmd = ['virt-sandbox', '--name', name] @@ -130,7 +129,7 @@ def run(args): params.append('-N') params.append(networkArgs) - allEnvs = source.get_env(args.template, args.template_dir) + allEnvs = source.get_env(tmpl, args.template_dir) envArgs = args.env if envArgs is not None: allEnvs = allEnvs + envArgs @@ -143,7 +142,7 @@ def run(args): else: pass - allVolumes = source.get_volumes(args.template, args.template_dir) + allVolumes = source.get_volumes(tmpl, args.template_dir) volumeArgs = args.volume if volumeArgs is not None: allVolumes = allVolumes + volumeArgs @@ -172,7 +171,7 @@ def run(args): def requires_template(parser): parser.add_argument("template", - help=_("name of the template")) + help=_("URI of the template")) def requires_name(parser): parser.add_argument("-n","--name", @@ -184,23 +183,10 @@ def check_connect(connectstr): raise ValueError("URI '%s' is not supported by virt-sandbox-image" % connectstr) return True -def requires_source(parser): - parser.add_argument("-s","--source", - default="docker", - help=_("name of the template")) - def requires_connect(parser): parser.add_argument("-c","--connect", help=_("Connect string for libvirt")) -def requires_auth_conn(parser): - parser.add_argument("-r","--registry", - help=_("Url of the custom registry")) - parser.add_argument("-u","--username", - help=_("Username for the custom registry")) - parser.add_argument("-p","--password", - help=_("Password for the custom registry")) - def requires_template_dir(parser): global default_template_dir parser.add_argument("-t","--template-dir", @@ -217,8 +203,6 @@ def gen_download_args(subparser): parser = subparser.add_parser("download", help=_("Download template data")) requires_template(parser) - requires_source(parser) - requires_auth_conn(parser) requires_template_dir(parser) parser.set_defaults(func=download) @@ -226,7 +210,6 @@ def gen_delete_args(subparser): parser = subparser.add_parser("delete", help=_("Delete template data")) requires_template(parser) - requires_source(parser) requires_template_dir(parser) parser.set_defaults(func=delete) @@ -234,7 +217,6 @@ def gen_create_args(subparser): parser = subparser.add_parser("create", help=_("Create image from template data")) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) parser.add_argument("-f","--format", @@ -247,7 +229,6 @@ def gen_run_args(subparser): help=_("Run an already built image")) requires_name(parser) requires_template(parser) - requires_source(parser) requires_connect(parser) requires_template_dir(parser) requires_image_dir(parser) diff --git a/libvirt-sandbox/image/sources/DockerSource.py b/libvirt-sandbox/image/sources/DockerSource.py index ff950ce..f367c8f 100644 --- a/libvirt-sandbox/image/sources/DockerSource.py +++ b/libvirt-sandbox/image/sources/DockerSource.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -29,6 +30,8 @@ import os import subprocess import shutil import collections +import urlparse + class DockerConfParser(): @@ -55,12 +58,6 @@ class DockerConfParser(): class DockerSource(Source): - www_auth_username = None - www_auth_password = None - - def __init__(self): - self.default_index_server = "index.docker.io" - def _check_cert_validate(self): major = sys.version_info.major SSL_WARNING = "SSL certificates couldn't be validated by default. You need to have 2.7.9/3.4.3 or higher" @@ -70,43 +67,38 @@ class DockerSource(Source): if (major == 2 and sys.hexversion < py2_7_9_hexversion) or (major == 3 and sys.hexversion < py3_4_3_hexversion): sys.stderr.write(SSL_WARNING) - def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): - if registry is None: - registry = self.default_index_server - - if username is not None: - self.www_auth_username = username - self.www_auth_password = password - + def download_template(self, template, templatedir): self._check_cert_validate() - tag = "latest" - offset = templatename.find(':') - if offset != -1: - tag = templatename[offset + 1:] - templatename = templatename[0:offset] + try: - (data, res) = self._get_json(registry, "/v1/repositories/" + templatename + "/images", - {"X-Docker-Token": "true"}) + (data, res) = self._get_json(template, + None, + "/v1/repositories" + template.path + "/images", + {"X-Docker-Token": "true"}) except urllib2.HTTPError, e: - raise ValueError(["Image '%s' does not exist" % templatename]) + raise ValueError(["Image '%s' does not exist" % template]) registryendpoint = res.info().getheader('X-Docker-Endpoints') token = res.info().getheader('X-Docker-Token') checksums = {} for layer in data: pass - (data, res) = self._get_json(registryendpoint, "/v1/repositories/" + templatename + "/tags", - { "Authorization": "Token " + token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/repositories" + template.path + "/tags", + { "Authorization": "Token " + token }) cookie = res.info().getheader('Set-Cookie') + tag = template.params.get("tag", "latest") if not tag in data: - raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, templatename)]) + raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, template)]) imagetagid = data[tag] - (data, res) = self._get_json(registryendpoint, "/v1/images/" + imagetagid + "/ancestry", - { "Authorization": "Token "+token }) + (data, res) = self._get_json(template, + registryendpoint, + "/v1/images/" + imagetagid + "/ancestry", + { "Authorization": "Token "+ token }) if data[0] != imagetagid: raise ValueError(["Expected first layer id '%s' to match image id '%s'", @@ -126,8 +118,11 @@ class DockerSource(Source): datafile = layerdir + "/template.tar.gz" if not os.path.exists(jsonfile) or not os.path.exists(datafile): - res = self._save_data(registryendpoint, "/v1/images/" + layerid + "/json", - { "Authorization": "Token " + token }, jsonfile) + res = self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/json", + { "Authorization": "Token " + token }, + jsonfile) createdFiles.append(jsonfile) layersize = int(res.info().getheader("Content-Length")) @@ -136,12 +131,15 @@ class DockerSource(Source): if layerid in checksums: datacsum = checksums[layerid] - self._save_data(registryendpoint, "/v1/images/" + layerid + "/layer", - { "Authorization": "Token "+token }, datafile, datacsum, layersize) + self._save_data(template, + registryendpoint, + "/v1/images/" + layerid + "/layer", + { "Authorization": "Token "+token }, + datafile, datacsum, layersize) createdFiles.append(datafile) index = { - "name": templatename, + "name": template.path, } indexfile = templatedir + "/" + imagetagid + "/index.json" @@ -160,9 +158,11 @@ class DockerSource(Source): shutil.rmtree(d) except: pass - def _save_data(self,server, path, headers, dest, checksum=None, datalen=None): + + def _save_data(self, template, server, path, headers, + dest, checksum=None, datalen=None): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers) csum = None if checksum is not None: @@ -201,8 +201,22 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise - def _get_url(self,server, path, headers): - url = "https://" + server + path + def _get_url(self, template, server, path, headers): + if template.protocol is None: + protocol = "https" + else: + protocol = template.protocol + + if server is None: + if template.hostname is None: + server = "index.docker.io" + else: + if template.port is not None: + server = template.hostname + ":" + template.port + else: + server = template.hostname + + url = urlparse.urlunparse((protocol, server, path, None, None, None)) debug("Fetching %s..." % url) req = urllib2.Request(url=url) @@ -212,16 +226,18 @@ class DockerSource(Source): req.add_header(h, headers[h]) #www Auth header starts - if self.www_auth_username is not None: - base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '') + if template.username and template.password: + base64string = base64.encodestring( + '%s:%s' % (template.username, + template.password)).replace('\n', '') req.add_header("Authorization", "Basic %s" % base64string) #www Auth header finish return urllib2.urlopen(req) - def _get_json(self,server, path, headers): + def _get_json(self, template, server, path, headers): try: - res = self._get_url(server, path, headers) + res = self._get_url(template, server, path, headers) data = json.loads(res.read()) debug("OK\n") return (data, res) @@ -229,11 +245,11 @@ class DockerSource(Source): debug("FAIL %s\n" % str(e)) raise - def create_template(self, templatename, templatedir, connect=None, format=None): + def create_template(self, template, templatedir, connect=None, format=None): if format is None: format = self.default_disk_format self._check_disk_format(format) - imagelist = self._get_image_list(templatename,templatedir) + imagelist = self._get_image_list(template, templatedir) imagelist.reverse() parentImage = None @@ -260,7 +276,7 @@ class DockerSource(Source): if not format in supportedFormats: raise ValueError(["Unsupported image format %s" % format]) - def _get_image_list(self,templatename,destdir): + def _get_image_list(self, template, destdir): imageparent = {} imagenames = {} imagedirs = os.listdir(destdir) @@ -273,13 +289,13 @@ class DockerSource(Source): jsonfile = destdir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) - parent = template.get("parent",None) + data = json.load(f) + parent = data.get("parent",None) if parent: imageparent[imagetagid] = parent - if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) - imagetagid = imagenames[templatename] + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path]) + imagetagid = imagenames[template.path] imagelist = [] while imagetagid != None: imagelist.append(imagetagid) @@ -318,7 +334,7 @@ class DockerSource(Source): cmd = cmd + params subprocess.call(cmd) - def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): imageusage = {} imageparent = {} imagenames = {} @@ -332,9 +348,9 @@ class DockerSource(Source): jsonfile = templatedir + "/" + imagetagid + "/template.json" if os.path.exists(jsonfile): with open(jsonfile,"r") as f: - template = json.load(f) + data = json.load(f) - parent = template.get("parent",None) + parent = data.get("parent",None) if parent: if parent not in imageusage: imageusage[parent] = [] @@ -342,10 +358,10 @@ class DockerSource(Source): imageparent[imagetagid] = parent - if not templatename in imagenames: - raise ValueError(["Image %s does not exist locally" %templatename]) + if not template.path in imagenames: + raise ValueError(["Image %s does not exist locally" % template.path]) - imagetagid = imagenames[templatename] + imagetagid = imagenames[template.path] while imagetagid != None: debug("Remove %s\n" % imagetagid) parent = imageparent.get(imagetagid,None) @@ -368,15 +384,15 @@ class DockerSource(Source): parent = None imagetagid = parent - def _get_template_data(self, templatename, templatedir): - imageList = self._get_image_list(templatename, templatedir) + def _get_template_data(self, template, templatedir): + imageList = self._get_image_list(template, templatedir) toplayer = imageList[0] diskfile = templatedir + "/" + toplayer + "/template.qcow2" configfile = templatedir + "/" + toplayer + "/template.json" return configfile, diskfile - def get_disk(self,templatename, templatedir, imagedir, sandboxname): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_disk(self, template, templatedir, imagedir, sandboxname): + configfile, diskfile = self._get_template_data(template, templatedir) tempfile = imagedir + "/" + sandboxname + ".qcow2" if not os.path.exists(imagedir): os.makedirs(imagedir) @@ -387,8 +403,8 @@ class DockerSource(Source): subprocess.call(cmd) return tempfile - def get_command(self, templatename, templatedir, userargs): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_command(self, template, templatedir, userargs): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) cmd = configParser.getCommand() entrypoint = configParser.getEntrypoint() @@ -401,13 +417,13 @@ class DockerSource(Source): else: return entrypoint + cmd - def get_env(self, templatename, templatedir): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_env(self, template, templatedir): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) return configParser.getEnvs() - def get_volumes(self, templatename, templatedir): - configfile, diskfile = self._get_template_data(templatename, templatedir) + def get_volumes(self, template, templatedir): + configfile, diskfile = self._get_template_data(template, templatedir) configParser = DockerConfParser(configfile) return configParser.getVolumes() diff --git a/libvirt-sandbox/image/sources/Source.py b/libvirt-sandbox/image/sources/Source.py index f1dd3e7..bb69682 100644 --- a/libvirt-sandbox/image/sources/Source.py +++ b/libvirt-sandbox/image/sources/Source.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. +# Copyright (C) 2015 Red Hat, Inc # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -33,14 +34,10 @@ class Source(): pass @abstractmethod - def download_template(self, templatename, templatedir, - registry=None, username=None, password=None): + def download_template(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template - :param registry: optional hostname of image registry server - :param username: optional username to authenticate against registry server - :param password: optional password to authenticate against registry server Download a template from the registry, storing it in the local filesystem @@ -48,10 +45,10 @@ class Source(): pass @abstractmethod - def create_template(self, templatename, templatedir, + def create_template(self, template, templatedir, connect=None, format=None): """ - :param templatename: name of the template image to create + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to store the template :param connect: libvirt connection URI :param format: disk image format @@ -63,9 +60,9 @@ class Source(): pass @abstractmethod - def delete_template(self, templatename, templatedir): + def delete_template(self, template, templatedir): """ - :param templatename: name of the template image to delete + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path from which to delete template Delete all local files associated with the template @@ -73,9 +70,9 @@ class Source(): pass @abstractmethod - def get_command(self, templatename, templatedir, userargs): + def get_command(self, template, templatedir, userargs): """ - :param templatename: name of the template image to query + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which templates are stored :param userargs: user specified arguments to run @@ -85,9 +82,9 @@ class Source(): pass @abstractmethod - def get_disk(self,templatename, templatedir, imagedir, sandboxname): + def get_disk(self, template, templatedir, imagedir, sandboxname): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template :param imagedir: local directory in which to storage disk image @@ -97,9 +94,9 @@ class Source(): pass @abstractmethod - def get_env(self,templatename, templatedir): + def get_env(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template Get the dict of environment variables to set @@ -107,9 +104,9 @@ class Source(): pass @abstractmethod - def get_volumes(self,templatename, templatedir): + def get_volumes(self, template, templatedir): """ - :param templatename: name of the template image to download + :param template: libvirt_sandbox.template.Template object :param templatedir: local directory path in which to find template Get the list of volumes associated with the template diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py new file mode 100644 index 0000000..9021242 --- /dev/null +++ b/libvirt-sandbox/image/template.py @@ -0,0 +1,109 @@ +# +# -*- coding: utf-8 -*- +# Authors: Daniel P. Berrange <berrange(a)redhat.com> +# +# Copyright (C) 2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# + +import urlparse +import importlib + +class Template(object): + + def __init__(self, + source, protocol, + hostname, port, + username, password, + path, params): + """ + :param source: template source name + :param protocol: network transport protocol or None + :param hostname: registry hostname or None + :param port: registry port or None + :param username: username or None + :param password: password or None + :param path: template path identifier + :param params: template parameters + + docker:///ubuntu + + docker+https://index.docker.io/ubuntu?tag=latest + """ + + self.source = source + self.protocol = protocol + self.hostname = hostname + self.port = port + self.username = username + self.password = password + self.path = path + self.params = params + if self.params is None: + self.params = {} + + def get_source_impl(self): + mod = importlib.import_module( + "libvirt_sandbox.image.sources." + + self.source.capitalize() + "Source") + classname = self.source.capitalize() + "Source" + classimpl = getattr(mod, classname) + return classimpl() + + def __repr__(self): + if self.protocol is not None: + scheme = self.source + "+" + self.protocol + else: + scheme = self.source + if self.hostname: + if self.port: + netloc = self.hostname + ":" + self.port + else: + netloc = self.hostname + + if self.username: + if self.password: + auth = self.username + ":" + self.password + else: + auth = self.username + netloc = auth + "@" + netloc + else: + netloc = None + + query = "&".join([key + "=" + self.params[key] for key in self.params.keys()]) + return urlparse.urlunparse((scheme, netloc, self.path, None, query, None)) + + @classmethod + def from_uri(klass, uri): + o = urlparse.urlparse(uri) + + idx = o.scheme.find("+") + if idx == -1: + source = o.scheme + protocol = None + else: + source = o.scheme[0:idx] + protocol = o.schema[idx + 1:] + + query = {} + for param in o.query.split("&"): + (key, val) = param.split("=") + query[key] = val + return klass(source, protocol, + o.hostname, o.port, + o.username, o.password, + o.path, query) + -- 2.4.3

1 0

[libvirt] [PATCH] qemu: Update balloon state after migration finishes
by Peter Krempa 21 Sep '15

21 Sep '15

Since qemu doesn't know at the beginning of migration what the actual balloon size is until the migration stream transfers the appropriate fields we also need to update the balloon size unconditionally in the finish phase of the migration. --- src/qemu/qemu_migration.c | 4 ++++ src/qemu/qemu_process.c | 2 +- src/qemu/qemu_process.h | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 903612b..38649ed 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -5701,6 +5701,10 @@ qemuMigrationFinish(virQEMUDriverPtr driver, if (qemuMigrationStopNBDServer(driver, vm, mig) < 0) goto endjob; + if (qemuProcessRefreshBalloonState(driver, vm, + QEMU_ASYNC_JOB_MIGRATION_IN) < 0) + goto endjob; + if (flags & VIR_MIGRATE_PERSIST_DEST) { if (qemuMigrationPersist(driver, vm, mig, !v3proto) < 0) { /* Hmpf. Migration was successful, but making it persistent diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 7187dc1..f14582b 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -2091,7 +2091,7 @@ qemuProcessReconnectRefreshChannelVirtioState(virQEMUDriverPtr driver, } -static int +int qemuProcessRefreshBalloonState(virQEMUDriverPtr driver, virDomainObjPtr vm, int asyncJob) diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h index d40f68d..b198c2b 100644 --- a/src/qemu/qemu_process.h +++ b/src/qemu/qemu_process.h @@ -115,4 +115,8 @@ virDomainDiskDefPtr qemuProcessFindDomainDiskByAlias(virDomainObjPtr vm, int qemuConnectAgent(virQEMUDriverPtr driver, virDomainObjPtr vm); +int qemuProcessRefreshBalloonState(virQEMUDriverPtr driver, + virDomainObjPtr vm, + int asyncJob); + #endif /* __QEMU_PROCESS_H__ */ -- 2.4.5

2 1

[libvirt] [PATCH 0/4] Remove open coding virProcessWait for root-squash
by John Ferlan 21 Sep '15

21 Sep '15

A followup of sorts to recently pushed patches regarding NFS root-squash. During libvirt-security list review it was pointed out that the new code was essentially open coding what virProcessWait does. However, since the model being used also was open coded and there was a time element, the change was allowed as is with the expectation that a cleanup patch would follow. Which is what leads into this series.... The series started out purely as removing the open code and replacing with the call to virProcessWait, but during that exercise I also realized that it was possible to create a 'netdir' in a NFS root-squash environment (eg, virDirCreate); however, the corrollary to remove the directory using a fork/exec didn't exist - in fact all that was called was rmdir, which failed to delete in the NFS root-squash environment. Rather than having a whole new interface, the first patch reworks virFileUnlink to check whether the target is a directory or a file and either call rmdir or unlink appropriately. The one common thread amongst the 3 API's changed here is they each looked to return an errno value, while typically virProcessWait consumers only return -1 and errno. Determining which failure in virProcessWait returns -1 is possible because one exit path uses virReportSystemError to report the error that caused the waitpid() to fail, while the other error path either receives the errno from the child process or if not present had already "assumed" EACCES, so these changes follow that model, except that if it's determined the waitpid failed, EINTR is returned similar to how virFileAccessibleAs sets errno and returns -1. John Ferlan (4): storage: Use virFileUnlink instead of rmdir virfile: Use virProcessWait in virFileOpenForked virfile: Use virProcessWait in virFileUnlink virfile: Use virProcessWait in virDirCreate src/storage/storage_backend_fs.c | 20 +++-- src/util/virfile.c | 153 ++++++++++++++++----------------------- 2 files changed, 71 insertions(+), 102 deletions(-) -- 2.1.0

2 9

[libvirt] [PATCH] vz: remove error logging from prlsdkUUIDParse
by Maxim Nestratov 21 Sep '15

21 Sep '15

From: Maxim Nestratov <mnestratov(a)virtuozzo.com> As far as not every call of prlsdkUUIDParse assume correct UUID supplied, there is no use to complain about wrong format in it. Otherwise our log is flooded with false error messages. For instance, calling prlsdkUUIDParse from prlsdkEventsHandler works as a filter and in case of uuid absence for event issuer, we simply know that we shouldn't continue further processing. Instead of error logging for all calls we should explicitly take into accaunt where it is called from. Signed-off-by: Maxim Nestratov <mnestratov(a)virtuozzo.com> --- src/vz/vz_sdk.c | 11 +++++++---- 1 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c index 744b58a..32ca1ef 100644 --- a/src/vz/vz_sdk.c +++ b/src/vz/vz_sdk.c @@ -329,8 +329,6 @@ prlsdkUUIDParse(const char *uuidstr, unsigned char *uuid) /* trim curly braces */ if (virUUIDParse(tmp + 1, uuid) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("UUID in config file malformed")); ret = -1; goto error; } @@ -365,8 +363,11 @@ prlsdkGetDomainIds(PRL_HANDLE sdkdom, PrlVmCfg_GetUuid(sdkdom, uuidstr, &len); prlsdkCheckRetGoto(pret, error); - if (prlsdkUUIDParse(uuidstr, uuid) < 0) + if (prlsdkUUIDParse(uuidstr, uuid) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Domain UUID is malformed or empty")); goto error; + } return 0; @@ -1724,8 +1725,10 @@ prlsdkEventsHandler(PRL_HANDLE prlEvent, PRL_VOID_PTR opaque) pret = PrlEvent_GetType(prlEvent, &prlEventType); prlsdkCheckRetGoto(pret, cleanup); - if (prlsdkUUIDParse(uuidstr, uuid) < 0) + if (prlsdkUUIDParse(uuidstr, uuid) < 0) { + VIR_DEBUG("Skipping event type %d", prlEventType); goto cleanup; + } switch (prlEventType) { case PET_DSP_EVT_VM_STATE_CHANGED: -- 1.7.1

4 4

[libvirt] [PATCH] vz: set mount point for container image-based disks
by Maxim Nestratov 21 Sep '15

21 Sep '15

From: Maxim Nestratov <mnestratov(a)virtuozzo.com> In order to support not only root disks with type=file for containers, we need to specify mount points for them. For instance, if a secondary disk is added by the following record in xml: <disk type='file' device='disk'> <driver type='ploop' cache='writeback'/> <source file='/vz/some_path_to_image_dir'/> <target bus='sata' dev='sdb'/> </disk> we are going to add it to container mounted to '/mnt/sdb' path. Signed-off-by: Maxim Nestratov <mnestratov(a)virtuozzo.com> --- src/vz/vz_sdk.c | 35 +++++++++++++++++++++++++++++++---- 1 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c index 744b58a..6642933 100644 --- a/src/vz/vz_sdk.c +++ b/src/vz/vz_sdk.c @@ -3292,21 +3292,48 @@ static int prlsdkAddDisk(PRL_HANDLE sdkdom, goto cleanup; } - if (bootDisk == true) { + if (bootDisk) { pret = PrlVmDev_GetIndex(sdkdisk, &devIndex); prlsdkCheckRetGoto(pret, cleanup); if (prlsdkAddDeviceToBootList(sdkdom, devIndex, devType, 0) < 0) goto cleanup; - /* If we add physical device as a boot disk to container - * we have to specify mount point for it */ - if (isCt) { + } + + if (isCt) { + + /* If we add a disk as a boot disk to a container + * (it doesn't matter whether it has image or real device backend) + * we have to specify the root mount point for it */ + if (bootDisk) { pret = PrlVmDevHd_SetMountPoint(sdkdisk, "/"); prlsdkCheckRetGoto(pret, cleanup); + } else { + + /* In case of image we need to specify a mount point + * to make it usable within a container */ + if (disk->src->type == VIR_STORAGE_TYPE_FILE) { + char *mount_point; + char *dev_name = strrchr(disk->dst, '/'); + + if (!dev_name) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Unexpected format of disk destination '%s'.")); + goto cleanup; + } + + if (virAsprintf(&mount_point, "/mnt%s", dev_name) < 0) + goto cleanup; + + pret = PrlVmDevHd_SetMountPoint(sdkdisk, mount_point); + VIR_FREE(mount_point); + prlsdkCheckRetGoto(pret, cleanup); + } } } + return 0; cleanup: PrlHandle_Free(sdkdisk); -- 1.7.1

3 4

[libvirt] [PATCH] Add a new test case for setUserPassword
by Luyao Huang 21 Sep '15

21 Sep '15

Signed-off-by: Luyao Huang <lhuang(a)redhat.com> --- cases/linux_domain.conf | 22 ++++++++ repos/domain/set_user_passwd.py | 111 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 repos/domain/set_user_passwd.py diff --git a/cases/linux_domain.conf b/cases/linux_domain.conf index 19daded..9a65cf3 100644 --- a/cases/linux_domain.conf +++ b/cases/linux_domain.conf @@ -299,6 +299,28 @@ domain:info_iothread conn qemu:///system +domain:set_user_passwd + guestname + $defaultname + username + $username + userpassword + $password + conn + qemu:///system + +domain:set_user_passwd + guestname + $defaultname + username + $username + userpassword + $password + conn + qemu:///system + flags + encrypted + domain:destroy guestname $defaultname diff --git a/repos/domain/set_user_passwd.py b/repos/domain/set_user_passwd.py new file mode 100644 index 0000000..bf9bd06 --- /dev/null +++ b/repos/domain/set_user_passwd.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python + +import libvirt +from libvirt import libvirtError +import lxml +import lxml.etree +import crypt +from utils import utils + +required_params = ('guestname', 'username', 'userpassword',) +optional_params = {'conn': 'qemu:///system', 'flags': '',} + +def get_guest_mac(vm): + tree = lxml.etree.fromstring(vm.XMLDesc(0)) + set = tree.xpath("/domain/devices/interface/mac") + + for n in set: + return n.attrib['address'] + + return False + + +def check_agent_status(vm): + """ make sure agent is okay to use """ + + tree = lxml.etree.fromstring(vm.XMLDesc(0)) + + set = tree.xpath("//channel[@type='unix']/target[@name='org.qemu.guest_agent.0']") + for n in set: + if n.attrib['state'] == 'connected': + return True + + return False + +def create_new_user(ipaddr, newusername, username, userpasswd, logger): + cmd = "useradd %s" % newusername + ret, retinfo = utils.remote_exec_pexpect(ipaddr, username, userpasswd, cmd) + if ret == 0 or "already exists" in retinfo: + return 0 + else: + logger.error("Fail: cannot create a new user: %s" % retinfo) + return 1 + +def verify_cur_user(ipaddr, username, userpasswd): + cmd = "whoami" + ret, retinfo = utils.remote_exec_pexpect(ipaddr, username, userpasswd, cmd) + + return ret + +def set_user_passwd(params): + """ + test API for setUserPassword in class virDomain + """ + + logger = params['logger'] + guest = params['guestname'] + username = params['username'] + userpasswd = params['userpassword'] + + if 'flags' in params: + if params['flags'] == 'encrypted': + flags = libvirt.VIR_DOMAIN_PASSWORD_ENCRYPTED + else: + flags = 0 + else: + flags = 0 + + try: + if 'conn' in params: + conn = libvirt.open(params['conn']) + else: + conn = libvirt.open(optional_params['conn']) + + logger.info("get connection to libvirtd") + vm = conn.lookupByName(guest) + logger.info("test guest name: %s" % guest) + + if not check_agent_status(vm): + logger.error("guest agent is not connected") + return 1 + + mac = get_guest_mac(vm) + if not mac: + logger.error("cannot get guest interface mac") + return 1 + + ipaddr = utils.mac_to_ip(mac, 180) + if not ipaddr: + logger.error("cannot get guest IP") + return 1 + + if flags > 0: + passwd = crypt.crypt("123456", crypt.mksalt(crypt.METHOD_SHA512)) + else: + passwd = "123456" + + + if create_new_user(ipaddr, "usertestapi", username, userpasswd, logger) != 0: + return 1 + + vm.setUserPassword("usertestapi", passwd, flags) + + if verify_cur_user(ipaddr, "usertestapi", "123456") != 0: + logger.error("cannot login guest via new user") + return 1 + + except libvirtError, e: + logger.error("API error message: %s" % e.message) + return 1 + + return 0 -- 1.8.3.1

1 1

[libvirt] [PATCH v2 0/7] Allow startupPolicy change on update-device
by Michal Privoznik 21 Sep '15

21 Sep '15

So, as Peter pointed out, we may want to updated startupPolicy for live domains too. That's what patches 2/7-7/7 do. Michal Privoznik (7): qemuDomainUpdateDeviceConfig: Allow startupPolicy update, yet again qemu: s/qemuDomainChangeDiskMediaLive/qemuDomainChangeDiskLive/ qemu_domain: Introduce qemuDomainDiskSourceDiffers qemuDomainChangeDiskLive: rework slightly qemu: s/virDomainDiskDiffersSourceOnly/qemuDomainDiskChangeSupported/ qemuDomainDiskChangeSupported: Fill in missing checks qemuDomainChangeDiskLive: Allow startupPolicy change src/conf/domain_conf.c | 127 ---------------------------------- src/conf/domain_conf.h | 2 - src/libvirt_private.syms | 1 - src/qemu/qemu_domain.c | 176 +++++++++++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_domain.h | 7 ++ src/qemu/qemu_driver.c | 80 ++++++++++++--------- 6 files changed, 232 insertions(+), 161 deletions(-) -- 2.4.6

2 10

[libvirt] [libvirt-php] error compiling on OS X 10.11 GM
by Andy Blyler 19 Sep '15

19 Sep '15

I’m attempting to compile the HEAD of the git repo on OS X 10.11 GM and I’m getting the following error: /Applications/Xcode.app/Contents/Developer/usr/bin/make all-recursive Making all in src /bin/sh ../libtool --tag=CC --mode=link clang -I/usr/local/Cellar/php56/5.6.13_2/include/php -I/usr/local/Cellar/php56/5.6.13_2/include/php/main -I/usr/local/Cellar/php56/5.6.13_2/include/php/TSRM -I/usr/local/Cellar/php56/5.6.13_2/include/php/Zend -I/usr/local/Cellar/php56/5.6.13_2/include/php/ext -I/usr/local/Cellar/php56/5.6.13_2/include/php/ext/date/lib -I/usr/local/Cellar/libxml2/2.9.2/include/libxml2 -I/usr/local/Cellar/libvirt/1.2.19/include -I/usr/local/Cellar/libvirt/1.2.19/include -DHAVE_CONFIG_H -I../winsrc -DCOMPILE_DL_LIBVIRT=1 -g -O2 -L/usr/local/Cellar/libxml2/2.9.2/lib -lxml2 -L/usr/local/Cellar/libvirt/1.2.19/lib -lvirt -L/usr/local/Cellar/libvirt/1.2.19/lib -lvirt-qemu -lvirt -o libvirt-php.la -rpath /usr/local/Cellar/php56-libvirt/HEAD/lib libvirt_php_la-vncfunc.lo libvirt_php_la-sockets.lo libvirt_php_la-libvirt-php.lo libtool: link: clang -dynamiclib -Wl,-undefined -Wl,dynamic_lookup -o .libs/libvirt-php.0.dylib .libs/libvirt_php_la-vncfunc.o .libs/libvirt_php_la-sockets.o .libs/libvirt_php_la-libvirt-php.o -L/usr/local/Cellar/libxml2/2.9.2/lib -lxml2 -L/usr/local/Cellar/libvirt/1.2.19/lib -lvirt-qemu -lvirt -g -O2 -install_name /usr/local/Cellar/php56-libvirt/HEAD/lib/libvirt-php.0.dylib -compatibility_version 1 -current_version 1.0 -Wl,-single_module duplicate symbol _gdebug in: .libs/libvirt_php_la-vncfunc.o .libs/libvirt_php_la-sockets.o duplicate symbol _gdebug in: .libs/libvirt_php_la-vncfunc.o .libs/libvirt_php_la-libvirt-php.o ld: 2 duplicate symbols for architecture x86_64 clang: error: linker command failed with exit code 1 (use -v to see invocation) My environment is: OS X: 10.11-x86_64 Xcode: 7.0 CLT: 7.0.0.0.1.1441394355 Clang: 7.0 build 700 X11: N/A

1 0

[libvirt] [libvirt-php] error compiling on OS X 10.11 GM
by Andy Blyler 19 Sep '15

19 Sep '15

I’m attempting to compile the HEAD of the git repo on OS X 10.11 GM and I’m getting the following error: /Applications/Xcode.app/Contents/Developer/usr/bin/make all-recursive Making all in src /bin/sh ../libtool --tag=CC --mode=link clang -I/usr/local/Cellar/php56/5.6.13_2/include/php -I/usr/local/Cellar/php56/5.6.13_2/include/php/main -I/usr/local/Cellar/php56/5.6.13_2/include/php/TSRM -I/usr/local/Cellar/php56/5.6.13_2/include/php/Zend -I/usr/local/Cellar/php56/5.6.13_2/include/php/ext -I/usr/local/Cellar/php56/5.6.13_2/include/php/ext/date/lib -I/usr/local/Cellar/libxml2/2.9.2/include/libxml2 -I/usr/local/Cellar/libvirt/1.2.19/include -I/usr/local/Cellar/libvirt/1.2.19/include -DHAVE_CONFIG_H -I../winsrc -DCOMPILE_DL_LIBVIRT=1 -g -O2 -L/usr/local/Cellar/libxml2/2.9.2/lib -lxml2 -L/usr/local/Cellar/libvirt/1.2.19/lib -lvirt -L/usr/local/Cellar/libvirt/1.2.19/lib -lvirt-qemu -lvirt -o libvirt-php.la -rpath /usr/local/Cellar/php56-libvirt/HEAD/lib libvirt_php_la-vncfunc.lo libvirt_php_la-sockets.lo libvirt_php_la-libvirt-php.lo libtool: link: clang -dynamiclib -Wl,-undefined -Wl,dynamic_lookup -o .libs/libvirt-php.0.dylib .libs/libvirt_php_la-vncfunc.o .libs/libvirt_php_la-sockets.o .libs/libvirt_php_la-libvirt-php.o -L/usr/local/Cellar/libxml2/2.9.2/lib -lxml2 -L/usr/local/Cellar/libvirt/1.2.19/lib -lvirt-qemu -lvirt -g -O2 -install_name /usr/local/Cellar/php56-libvirt/HEAD/lib/libvirt-php.0.dylib -compatibility_version 1 -current_version 1.0 -Wl,-single_module duplicate symbol _gdebug in: .libs/libvirt_php_la-vncfunc.o .libs/libvirt_php_la-sockets.o duplicate symbol _gdebug in: .libs/libvirt_php_la-vncfunc.o .libs/libvirt_php_la-libvirt-php.o ld: 2 duplicate symbols for architecture x86_64 clang: error: linker command failed with exit code 1 (use -v to see invocation) My environment is: OS X: 10.11-x86_64 Xcode: 7.0 CLT: 7.0.0.0.1.1441394355 Clang: 7.0 build 700 X11: N/A

1 0

[libvirt] [PATCH] libxl: fix AttachDeviceConfig on hostdev type
by Chunyan Liu 18 Sep '15

18 Sep '15

After attach-device a <hostdev> with --config, new device doesn't show up in dumpxml and in guest. To fix that, set dev->data.hostdev = NULL after work so that the pointer is not freed, since vmdef has the pointer and still need it. Signed-off-by: Chunyan Liu <cyliu(a)suse.com> --- src/libxl/libxl_driver.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c index e2797d5..8087c27 100644 --- a/src/libxl/libxl_driver.c +++ b/src/libxl/libxl_driver.c @@ -3312,6 +3312,7 @@ libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev) if (virDomainHostdevInsert(vmdef, hostdev) < 0) return -1; + dev->data.hostdev = NULL; break; default: -- 1.8.5.6

3 2

[libvirt] [PATCH v2 0/8] Simplify and fix qemuMigrationFinish
by Jiri Denemark 18 Sep '15

18 Sep '15

While hacking qemuMigrationFinish I found it pretty hard to follow and revealed few bugs (patches 3 to 5, and 8) in the code. Version 2: - rebased and review comments addressed - new patch "qemu: Fix some corner cases in persistent migration" Jiri Denemark (8): qemu: Split qemuMigrationFinish qemu: Simplify qemuMigrationFinish qemu: Don't fail migration on save status failure qemu: Kill domain when migration finish fails qemu: Don't report false errors in migration protocol v2 qemuDomainEventQueue: Check if event is non-NULL qemu: Queue events in migration Finish phase ASAP qemu: Fix some corner cases in persistent migration src/qemu/qemu_blockjob.c | 6 +- src/qemu/qemu_cgroup.c | 3 +- src/qemu/qemu_domain.c | 6 +- src/qemu/qemu_driver.c | 87 +++++++----------- src/qemu/qemu_hotplug.c | 26 +++--- src/qemu/qemu_migration.c | 223 ++++++++++++++++++++++++---------------------- src/qemu/qemu_process.c | 72 +++++---------- 7 files changed, 187 insertions(+), 236 deletions(-) -- 2.5.1

2 22

[libvirt] [PATCH v2 0/12] migration: support all toURI and proto combos
by Nikolay Shirokovskiy 18 Sep '15

18 Sep '15

Current implementation of 'toURI' migration interfaces does not support all combinations of interface versions and protocol versions. For example 'toURI2' with p2p flag will not migrate if driver supports only v3params proto. This is not convinient as drivers that starts to support migration have to manually support older versions of protocol. I guess this should be done in one place, namely here. Another issue is that there are a lot of code duplication in implementation of toURI interfaces and it is not obvious from code how are they related. This implementation uses extensible parameters as intermediate parameters representation. This is possible as interfaces are done backward compatible in terms of parameters and later versions supports all parameters of former versions. = Changes from version1 Patch is splitted into a set. Quite a big one as a result of the following strategy: 1. each change in behaviour even subtle one deserves a separate patch. One patch changes one aspect in behaviour. 2. separate pure refactoring steps into patches too as rather simple refactor steps could introduce many line changes. Mark such patches with 'refactor:' Now every patch is easy to grasp I think. The resulted cumulative patch is slightly different from first in behaviour but I'm not going to describe the differece here as original patch was not reviewed in details by anyone anyway ) src/libvirt-domain.c | 520 +++++++++++++++++++++----------------------------- 1 files changed, 216 insertions(+), 304 deletions(-)

4 47

Re: [libvirt] [Qemu-devel] internal snapshots with sheepdog
by Kevin Wolf 18 Sep '15

18 Sep '15

Am 18.09.2015 um 11:03 hat Vasiliy Tolstov geschrieben: > 2015-09-18 12:02 GMT+03:00 Kevin Wolf <kwolf(a)redhat.com>: > > Doesn't sheepdog already support storing snapshots in the same image? > > I thought it would just work; at least, there's some code there for it. > > Yes, qemu and sheepdog have ability to create internal snapshot, i > miss that, but when i'm try in libvirt create live snapshot - libvirt > says, that this is not possible with raw image. Then I guess your mail should be directed to the libvirt list instead (CCed). Kevin

2 1

[libvirt] [java] [PATCH] add missing virDomainSetMemoryStatsPeriod API
by Wei Mingzhi 18 Sep '15

18 Sep '15

Hi, The attached patch adds the missing virDomainSetMemoryStatsPeriod API, which was introduced since around 2013. Cheers, Wei Mingzhi -- "The best way to predict the future is to invent it." -Alan Kay

1 0

[libvirt] [PATCH] libxl: fix AttachDeviceConfig on hostdev type
by Chunyan Liu 17 Sep '15

17 Sep '15

After attach-device a <hostdev> with --config, new device doesn't show up in dumpxml and in guest. To fix that, set dev->data.hostdev = NULL after work so that the pointer is not freed, since vmdef has the pointer and still need it. Signed-off-by: Chunyan Liu <cyliu(a)suse.com> --- src/libxl/libxl_driver.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c index e2797d5..8087c27 100644 --- a/src/libxl/libxl_driver.c +++ b/src/libxl/libxl_driver.c @@ -3312,6 +3312,7 @@ libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev) if (virDomainHostdevInsert(vmdef, hostdev) < 0) return -1; + dev->data.hostdev = NULL; break; default: -- 1.8.5.6

2 1

[libvirt] VMware: Map vpx:// to dcPath
by Richard W.M. Jones 17 Sep '15

17 Sep '15

vpx:// paths looks like this: vpx://vcenter.example.com/MyFolder/MyDatacenter/MyCluster/esxi but to connect to the datastore to read the underlying disk image, libguestfs must form a URL like this: https://vcenter.example.com/folder/data/guest/guest-flat.vmdk?dcPath=MyFold… All parts of this URL can be worked out from the URL or the libvirt XML *except* the ?dcPath=... parameter. The problem is that dcPath isn't a straight mapping from the vpx:// path. The particular problem is that if there is a cluster name in the path (eg 'MyCluster') it appears that we have to remove it. ie: dcPath=MyFolder/MyDatacenter/MyCluster - does not work dcPath=MyFolder/MyDatacenter - works That would be OK if there was always a cluster name at the end of the path, but there isn't. Clusters are completely optional, and AFAIK you can't tell if something is a cluster path element just by examining the name, since clusters can be given arbitrary names by the vCenter admin. So: (1) Is there something I'm missing here? Maybe the libvirt VMware driver presents this information already and I'm just missing it? (2) Can we add some way to more easily map from vpx:// paths to datastore URLs? The whole process is very complex even without the ambiguity - it takes a couple of pages of code to do the mapping. Rich. -- Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones Read my programming and virtualization blog: http://rwmj.wordpress.com virt-top is 'top' for virtual machines. Tiny program with many powerful monitoring features, net stats, disk stats, logging, etc. http://people.redhat.com/~rjones/virt-top

3 8

[libvirt] [PATCH v3] qemu: Validate address type when attaching a disk device.
by Ruifeng Bian 17 Sep '15

17 Sep '15

Bug fix for: https://bugzilla.redhat.com/show_bug.cgi?id=1257844 Attach-device can hotplug a virtio disk device with any address type now, it need to validate the address type before the attachment. Attaching a disk device with --config option need to check address type. Otherwise, the domain cloudn't be started normally. This patch add a basic check for address type. Detaching a disk device with --config also need to check disk options, add qemuCheckDiskConfig() method in qemuDomainDetachDeviceConfig(). --- src/qemu/qemu_command.c | 44 +++++++++++++++++++++++++++++++++++++++++++- src/qemu/qemu_driver.c | 2 ++ src/qemu/qemu_hotplug.c | 22 ++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index ec5e3d4..a2c7483 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -3579,12 +3579,54 @@ qemuCheckDiskConfig(virDomainDiskDefPtr disk) goto error; } } + + if (disk->info.type) { + switch (disk->bus) { + case VIR_DOMAIN_DISK_BUS_IDE: + case VIR_DOMAIN_DISK_BUS_SCSI: + case VIR_DOMAIN_DISK_BUS_SATA: + case VIR_DOMAIN_DISK_BUS_FDC: + if (disk->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_DRIVE) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("disk cannot have an address of type '%s'"), + virDomainDeviceAddressTypeToString(disk->info.type)); + goto error; + } + break; + case VIR_DOMAIN_DISK_BUS_USB: + if (disk->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_USB) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("usb disk cannot have an address of type '%s'"), + virDomainDeviceAddressTypeToString(disk->info.type)); + goto error; + } + break; + case VIR_DOMAIN_DISK_BUS_VIRTIO: + if (disk->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW && + disk->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390 && + disk->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_MMIO && + disk->info.type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("virtio disk cannot have an address of type '%s'"), + virDomainDeviceAddressTypeToString(disk->info.type)); + goto error; + } + break; + case VIR_DOMAIN_DISK_BUS_XEN: + case VIR_DOMAIN_DISK_BUS_UML: + case VIR_DOMAIN_DISK_BUS_SD: + /* no address type currently, return false if address provided */ + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("disk cannot have an address of type '%s'"), + virDomainDeviceAddressTypeToString(disk->info.type)); + goto error; + } + } return 0; error: return -1; } - /* Check whether the device address is using either 'ccw' or default s390 * address format and whether that's "legal" for the current qemu and/or * guest os.machine type. This is the corollary to the code which doesn't diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 91eb661..61424b0 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -8205,6 +8205,8 @@ qemuDomainDetachDeviceConfig(virDomainDefPtr vmdef, switch ((virDomainDeviceType) dev->type) { case VIR_DOMAIN_DEVICE_DISK: disk = dev->data.disk; + if (qemuCheckDiskConfig(disk) < 0) + return -1; if (!(det_disk = virDomainDiskRemoveByName(vmdef, disk->dst))) { virReportError(VIR_ERR_INVALID_ARG, _("no target device %s"), disk->dst); diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c index 63fafa6..1f46647 100644 --- a/src/qemu/qemu_hotplug.c +++ b/src/qemu/qemu_hotplug.c @@ -335,6 +335,28 @@ qemuDomainAttachVirtioDiskDevice(virConnectPtr conn, if (!qemuCheckCCWS390AddressSupport(vm->def, disk->info, priv->qemuCaps, disk->dst)) goto cleanup; + + if (qemuDomainMachineIsS390CCW(vm->def) && + virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_VIRTIO_CCW)) { + if (!virDomainDeviceAddressIsValid(&disk->info, + VIR_DOMAIN_DEVICE_ADDRESS_TYPE_CCW)) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("device cannot be attached without a valid CCW address")); + goto cleanup; + } + } else if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_VIRTIO_S390)) { + if (!virDomainDeviceAddressIsValid(&disk->info, + VIR_DOMAIN_DEVICE_ADDRESS_TYPE_VIRTIO_S390)) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("device cannot be attached without a valid S390 address")); + goto cleanup; + } + } else if (!virDomainDeviceAddressIsValid(&disk->info, + VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI)) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("device cannot be attached without a valid PCI address")); + goto cleanup; + } } for (i = 0; i < vm->def->ndisks; i++) { -- 2.4.3

2 2

[libvirt] [PATCH v2] virfile: Check for existence of dir in virFileDeleteTree
by John Ferlan 16 Sep '15

16 Sep '15

Commit id 'f1f68ca33' added code to remove the directory paths for auto-generated sockets, but that code could be called before the paths were created resulting in generating error messages from virFileDeleteTree indicating that the file doesn't exist. Rather than "enforce" all callers to make the non-NULL and existence checks, modify the virFileDeleteTree API to silently ignore NULL on input and non-existent directory trees. Signed-off-by: John Ferlan <jferlan(a)redhat.com> --- Difference over v1: Add checks in virFileDeleteTree src/qemu/qemu_process.c | 6 ++---- src/util/virfile.c | 8 ++++++-- tests/virhostdevtest.c | 2 +- tests/virscsitest.c | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index ce2c70c..155846e 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -5269,14 +5269,12 @@ void qemuProcessStop(virQEMUDriverPtr driver, ignore_value(virAsprintf(&tmppath, "%s/domain-%s", cfg->libDir, vm->def->name)); - if (tmppath) - virFileDeleteTree(tmppath); + virFileDeleteTree(tmppath); VIR_FREE(tmppath); ignore_value(virAsprintf(&tmppath, "%s/domain-%s", cfg->channelTargetDir, vm->def->name)); - if (tmppath) - virFileDeleteTree(tmppath); + virFileDeleteTree(tmppath); VIR_FREE(tmppath); ignore_value(virDomainChrDefForeach(vm->def, diff --git a/src/util/virfile.c b/src/util/virfile.c index 75819d9..fe9f8d4 100644 --- a/src/util/virfile.c +++ b/src/util/virfile.c @@ -934,13 +934,17 @@ int virFileNBDDeviceAssociate(const char *file, */ int virFileDeleteTree(const char *dir) { - DIR *dh = opendir(dir); + DIR *dh; struct dirent *de; char *filepath = NULL; int ret = -1; int direrr; - if (!dh) { + /* Silently return 0 if passed NULL or directory doesn't exist */ + if (!dir || !virFileExists(dir)) + return 0; + + if (!(dh = opendir(dir))) { virReportSystemError(errno, _("Cannot open dir '%s'"), dir); return -1; diff --git a/tests/virhostdevtest.c b/tests/virhostdevtest.c index 1e93819..065b825 100644 --- a/tests/virhostdevtest.c +++ b/tests/virhostdevtest.c @@ -65,7 +65,7 @@ myCleanup(void) } if (mgr) { - if (mgr->stateDir && !getenv("LIBVIRT_SKIP_CLEANUP")) + if (!getenv("LIBVIRT_SKIP_CLEANUP")) virFileDeleteTree(mgr->stateDir); virObjectUnref(mgr->activePCIHostdevs); diff --git a/tests/virscsitest.c b/tests/virscsitest.c index a86ca28..88286f1 100644 --- a/tests/virscsitest.c +++ b/tests/virscsitest.c @@ -241,7 +241,7 @@ mymain(void) ret = -1; cleanup: - if (tmpdir && getenv("LIBVIRT_SKIP_CLEANUP") == NULL) + if (getenv("LIBVIRT_SKIP_CLEANUP") == NULL) virFileDeleteTree(tmpdir); VIR_FREE(virscsi_prefix); return ret; -- 2.1.0

2 1

[libvirt] [PATCH] qemu: Check for existence of auto-generated socket path before removing
by John Ferlan 16 Sep '15

16 Sep '15

Commit id 'f1f68ca33' added code to remove the directory paths for auto-generated sockets, but that code could be called before the paths were created resulting in generating error messages from virFileDeleteTree indicating that the file doesn't exist. So just add a check before attemping the directory delete for existence. Signed-off-by: John Ferlan <jferlan(a)redhat.com> --- src/qemu/qemu_process.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index ce2c70c..b55eb52 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -5269,13 +5269,13 @@ void qemuProcessStop(virQEMUDriverPtr driver, ignore_value(virAsprintf(&tmppath, "%s/domain-%s", cfg->libDir, vm->def->name)); - if (tmppath) + if (tmppath && virFileExists(tmppath)) virFileDeleteTree(tmppath); VIR_FREE(tmppath); ignore_value(virAsprintf(&tmppath, "%s/domain-%s", cfg->channelTargetDir, vm->def->name)); - if (tmppath) + if (tmppath && virFileExists(tmppath)) virFileDeleteTree(tmppath); VIR_FREE(tmppath); -- 2.1.0

3 3

[libvirt] [PATCH v2] virsh: Teach attach-interface to --print-xml
by Michal Privoznik 16 Sep '15

16 Sep '15

We have the same argument to many other commands that produce an XML based on what user typed. But unfortunately attach-interface was missing it. Maybe nobody had needed it yet. Well, I did just now. Signed-off-by: Michal Privoznik <mprivozn(a)redhat.com> --- Diff to v1: - Commit message grammar fix - Fixed usage of virDomain tools/virsh-domain.c | 28 ++++++++++++++++++++-------- tools/virsh.pod | 4 ++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index b029b65..ce84930 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -862,6 +862,10 @@ static const vshCmdOptDef opts_attach_interface[] = { .type = VSH_OT_BOOL, .help = N_("affect current domain") }, + {.name = "print-xml", + .type = VSH_OT_BOOL, + .help = N_("print XML document rather than attach the interface") + }, {.name = NULL} }; @@ -938,13 +942,6 @@ cmdAttachInterface(vshControl *ctl, const vshCmd *cmd) if (live) flags |= VIR_DOMAIN_AFFECT_LIVE; - if (!(dom = virshCommandOptDomain(ctl, cmd, NULL))) - return false; - - if (persistent && - virDomainIsActive(dom) == 1) - flags |= VIR_DOMAIN_AFFECT_LIVE; - if (vshCommandOptStringReq(ctl, cmd, "type", &type) < 0 || vshCommandOptStringReq(ctl, cmd, "source", &source) < 0 || vshCommandOptStringReq(ctl, cmd, "target", &target) < 0 || @@ -1051,6 +1048,7 @@ cmdAttachInterface(vshControl *ctl, const vshCmd *cmd) virBufferAddLit(&buf, "</bandwidth>\n"); } + virBufferAdjustIndent(&buf, -2); virBufferAddLit(&buf, "</interface>\n"); if (virBufferError(&buf)) { @@ -1060,6 +1058,19 @@ cmdAttachInterface(vshControl *ctl, const vshCmd *cmd) xml = virBufferContentAndReset(&buf); + if (vshCommandOptBool(cmd, "print-xml")) { + vshPrint(ctl, "%s", xml); + functionReturn = true; + goto cleanup; + } + + if (!(dom = virshCommandOptDomain(ctl, cmd, NULL))) + goto cleanup; + + if (persistent && + virDomainIsActive(dom) == 1) + flags |= VIR_DOMAIN_AFFECT_LIVE; + if (flags || current) ret = virDomainAttachDeviceFlags(dom, xml, flags); else @@ -1075,7 +1086,8 @@ cmdAttachInterface(vshControl *ctl, const vshCmd *cmd) } cleanup: - virDomainFree(dom); + if (dom) + virDomainFree(dom); virBufferFreeAndReset(&buf); return functionReturn; } diff --git a/tools/virsh.pod b/tools/virsh.pod index 83c445d3..0212e7a 100644 --- a/tools/virsh.pod +++ b/tools/virsh.pod @@ -2507,6 +2507,7 @@ Likewise, I<--shareable> is an alias for I<--mode shareable>. [[[I<--live>] [I<--config>] | [I<--current>]] | [I<--persistent>]] [I<--target target>] [I<--mac mac>] [I<--script script>] [I<--model model>] [I<--inbound average,peak,burst,floor>] [I<--outbound average,peak,burst>] +[I<--print-xml>] Attach a new network interface to the domain. I<type> can be I<network> to indicate connection via a libvirt virtual network, or @@ -2536,6 +2537,9 @@ kilobytes in a single burst at I<peak> speed as described in the Network XML documentation at L<http://libvirt.org/formatnetwork.html#elementQoS>. +If I<--print-xml> is specified, then the XML of the interface that would be +attached is printed instead. + If I<--live> is specified, affect a running domain. If I<--config> is specified, affect the next startup of a persistent domain. If I<--current> is specified, affect the current domain state. -- 2.4.6

2 1