[libvirt] [sandbox 0/6] Misc patches

Hi all, Here are a few patches I found sitting on my local copy. I also added a few patches to convert to python3. Cédric Bosdonnat (6): Pass debug and verbose values to init machine: use squash security mode for non-root virt-sandbox mounts Add tests .log and .trs files to gitignore service: fix bad ConfigMountHostImage constructor call Convert to python3 Don't hardcode interpreter path .gitignore | 2 + bin/virt-sandbox-image | 2 +- bin/virt-sandbox-service | 75 +++++++++++++---------- bin/virt-sandbox.c | 3 + examples/demo.py | 2 +- examples/shell.py | 2 +- examples/virt-sandbox-mkinitrd.py | 2 +- examples/virt-sandbox.py | 2 +- libvirt-sandbox/image/cli.py | 20 +++--- libvirt-sandbox/image/sources/base.py | 1 - libvirt-sandbox/image/sources/docker.py | 42 ++++++------- libvirt-sandbox/image/sources/virtbuilder.py | 1 - libvirt-sandbox/image/template.py | 8 +-- libvirt-sandbox/libvirt-sandbox-builder-machine.c | 5 +- libvirt-sandbox/libvirt-sandbox-config.c | 75 +++++++++++++++++++++++ libvirt-sandbox/libvirt-sandbox-config.h | 6 ++ libvirt-sandbox/libvirt-sandbox-init-common.c | 3 + libvirt-sandbox/libvirt-sandbox.sym | 4 ++ 18 files changed, 179 insertions(+), 76 deletions(-) -- 2.15.1

libvirt-sandbox-init-common is expecting -d and -v parameters to set it in debug or verbose mode... but those will never be passed by the launcher program. Writing the core.debug and core.verbose parameters in the sandbox configuration makes those values actually usable from the init. --- bin/virt-sandbox.c | 3 ++ libvirt-sandbox/libvirt-sandbox-config.c | 75 +++++++++++++++++++++++++++ libvirt-sandbox/libvirt-sandbox-config.h | 6 +++ libvirt-sandbox/libvirt-sandbox-init-common.c | 3 ++ libvirt-sandbox/libvirt-sandbox.sym | 4 ++ 5 files changed, 91 insertions(+) diff --git a/bin/virt-sandbox.c b/bin/virt-sandbox.c index 3058013..6032562 100644 --- a/bin/virt-sandbox.c +++ b/bin/virt-sandbox.c @@ -273,6 +273,9 @@ int main(int argc, char **argv) { if (shell) gvir_sandbox_config_set_shell(cfg, TRUE); + gvir_sandbox_config_set_debug(cfg, debug); + gvir_sandbox_config_set_verbose(cfg, verbose); + if (isatty(STDIN_FILENO)) gvir_sandbox_config_interactive_set_tty(icfg, TRUE); diff --git a/libvirt-sandbox/libvirt-sandbox-config.c b/libvirt-sandbox/libvirt-sandbox-config.c index 8709736..73a0fa4 100644 --- a/libvirt-sandbox/libvirt-sandbox-config.c +++ b/libvirt-sandbox/libvirt-sandbox-config.c @@ -68,6 +68,9 @@ struct _GVirSandboxConfigPrivate gchar *secLabel; gboolean secDynamic; + + gboolean debug; + gboolean verbose; }; G_DEFINE_ABSTRACT_TYPE(GVirSandboxConfig, gvir_sandbox_config, G_TYPE_OBJECT); @@ -1926,6 +1929,59 @@ gboolean gvir_sandbox_config_set_security_opts(GVirSandboxConfig *config, return ret; } +/** + * gvir_sandbox_config_set_debug: + * @config: (transfer none): the sandbox config + * @debug: true if the container init should print debugging messages + * + * Set whether the container init should print debugging messages. + */ +void gvir_sandbox_config_set_debug(GVirSandboxConfig *config, gboolean debug) +{ + GVirSandboxConfigPrivate *priv = config->priv; + priv->debug = debug; +} + +/** + * gvir_sandbox_config_get_debug: + * @config: (transfer none): the sandbox config + * + * Retrieves the sandbox debug flag + * + * Returns: the debug flag + */ +gboolean gvir_sandbox_config_get_debug(GVirSandboxConfig *config) +{ + GVirSandboxConfigPrivate *priv = config->priv; + return priv->debug; +} + +/** + * gvir_sandbox_config_set_verbose: + * @config: (transfer none): the sandbox config + * @verbose: true if the container init should be verbose + * + * Set whether the container init should be verbose. + */ +void gvir_sandbox_config_set_verbose(GVirSandboxConfig *config, gboolean verbose) +{ + GVirSandboxConfigPrivate *priv = config->priv; + priv->verbose = verbose; +} + +/** + * gvir_sandbox_config_get_verbose: + * @config: (transfer none): the sandbox config + * + * Retrieves the sandbox verbose flag + * + * Returns: the verbose flag + */ +gboolean gvir_sandbox_config_get_verbose(GVirSandboxConfig *config) +{ + GVirSandboxConfigPrivate *priv = config->priv; + return priv->verbose; +} static GVirSandboxConfigMount *gvir_sandbox_config_load_config_mount(GKeyFile *file, guint i, @@ -2415,6 +2471,22 @@ static gboolean gvir_sandbox_config_load_config(GVirSandboxConfig *config, priv->secDynamic = b; } + b = g_key_file_get_boolean(file, "core", "debug", &e); + if (e) { + g_error_free(e); + e = NULL; + } else { + priv->debug = b; + } + + b = g_key_file_get_boolean(file, "core", "verbose", &e); + if (e) { + g_error_free(e); + e = NULL; + } else { + priv->verbose = b; + } + ret = TRUE; cleanup: return ret; @@ -2677,6 +2749,9 @@ static void gvir_sandbox_config_save_config(GVirSandboxConfig *config, if (priv->secLabel) g_key_file_set_string(file, "security", "label", priv->secLabel); g_key_file_set_boolean(file, "security", "dynamic", priv->secDynamic); + + g_key_file_set_boolean(file, "core", "debug", priv->debug); + g_key_file_set_boolean(file, "core", "verbose", priv->verbose); } diff --git a/libvirt-sandbox/libvirt-sandbox-config.h b/libvirt-sandbox/libvirt-sandbox-config.h index e5e53f7..8950e25 100644 --- a/libvirt-sandbox/libvirt-sandbox-config.h +++ b/libvirt-sandbox/libvirt-sandbox-config.h @@ -180,6 +180,12 @@ gboolean gvir_sandbox_config_set_security_opts(GVirSandboxConfig *config, const gchar *optstr, GError**error); +void gvir_sandbox_config_set_debug(GVirSandboxConfig *config, gboolean debug); +gboolean gvir_sandbox_config_get_debug(GVirSandboxConfig *config); + +void gvir_sandbox_config_set_verbose(GVirSandboxConfig *config, gboolean verbose); +gboolean gvir_sandbox_config_get_verbose(GVirSandboxConfig *config); + gchar **gvir_sandbox_config_get_command(GVirSandboxConfig *config); G_END_DECLS diff --git a/libvirt-sandbox/libvirt-sandbox-init-common.c b/libvirt-sandbox/libvirt-sandbox-init-common.c index 7ea63cf..240ca83 100644 --- a/libvirt-sandbox/libvirt-sandbox-init-common.c +++ b/libvirt-sandbox/libvirt-sandbox-init-common.c @@ -1442,6 +1442,9 @@ int main(int argc, char **argv) { goto cleanup; } + debug = gvir_sandbox_config_get_debug(config); + verbose = gvir_sandbox_config_get_verbose(config); + setenv("PATH", "/bin:/usr/bin:/usr/local/bin:/sbin/:/usr/sbin", 1); unsetenv("LD_LIBRARY_PATH"); diff --git a/libvirt-sandbox/libvirt-sandbox.sym b/libvirt-sandbox/libvirt-sandbox.sym index b7c5921..1bead3e 100644 --- a/libvirt-sandbox/libvirt-sandbox.sym +++ b/libvirt-sandbox/libvirt-sandbox.sym @@ -120,6 +120,8 @@ LIBVIRT_SANDBOX_0.6.0 { gvir_sandbox_config_get_userid; gvir_sandbox_config_get_username; gvir_sandbox_config_get_uuid; + gvir_sandbox_config_get_debug; + gvir_sandbox_config_get_verbose; gvir_sandbox_config_find_mount; gvir_sandbox_config_has_networks; gvir_sandbox_config_has_mounts; @@ -143,6 +145,8 @@ LIBVIRT_SANDBOX_0.6.0 { gvir_sandbox_config_set_security_label; gvir_sandbox_config_set_security_opts; gvir_sandbox_config_set_uuid; + gvir_sandbox_config_set_debug; + gvir_sandbox_config_set_verbose; gvir_sandbox_config_initrd_add_module; gvir_sandbox_config_initrd_get_init; -- 2.15.1

On Tue, Dec 05, 2017 at 10:53:17AM +0100, Cédric Bosdonnat wrote:
libvirt-sandbox-init-common is expecting -d and -v parameters to set it in debug or verbose mode... but those will never be passed by the launcher program.
Hmm, libvirt-sandbox-init-{qemu,lxc} both know how to pass -d to init-common. If either of the virt specific init programs are in debug mode (as indicated by the word "debug" in kernel command line or equiv), then they pass -d to init-common. You're right that there's no way to pass -v though. Also, it might be useful to run init-common in debug mode without the virt specific init being in debug mode, to avoid drowning in debug.
Writing the core.debug and core.verbose parameters in the sandbox configuration makes those values actually usable from the init. --- bin/virt-sandbox.c | 3 ++ libvirt-sandbox/libvirt-sandbox-config.c | 75 +++++++++++++++++++++++++++ libvirt-sandbox/libvirt-sandbox-config.h | 6 +++ libvirt-sandbox/libvirt-sandbox-init-common.c | 3 ++ libvirt-sandbox/libvirt-sandbox.sym | 4 ++ 5 files changed, 91 insertions(+)
Reviewed-by: Daniel P. Berrange <berrange@redhat.com> if the commit message is fixed Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

On Tue, 2017-12-05 at 16:52 +0000, Daniel P. Berrange wrote:
On Tue, Dec 05, 2017 at 10:53:17AM +0100, Cédric Bosdonnat wrote:
libvirt-sandbox-init-common is expecting -d and -v parameters to set it in debug or verbose mode... but those will never be passed by the launcher program.
Hmm, libvirt-sandbox-init-{qemu,lxc} both know how to pass -d to init-common. If either of the virt specific init programs are in debug mode (as indicated by the word "debug" in kernel command line or equiv), then they pass -d to init-common.
Hum, then I should surely revise that commit to remove the debug-related code. Instead I should change the init-{lxc,qemu} to actually pass the debug flag to init-common.
You're right that there's no way to pass -v though.
Also, it might be useful to run init-common in debug mode without the virt specific init being in debug mode, to avoid drowning in debug.
Hum, then keeping the debug-related flag I introduced could help, but then I'll need to rename it to make it more explicit. Or maybe I should change the debug parameter to take a level value: 0: all (default) 1: only the init-common In which case the user-exposed options should be expanded to use these. I'll push the other commits, but not this one. -- Cedric
Writing the core.debug and core.verbose parameters in the sandbox configuration makes those values actually usable from the init. --- bin/virt-sandbox.c | 3 ++ libvirt-sandbox/libvirt-sandbox-config.c | 75 +++++++++++++++++++++++++++ libvirt-sandbox/libvirt-sandbox-config.h | 6 +++ libvirt-sandbox/libvirt-sandbox-init-common.c | 3 ++ libvirt-sandbox/libvirt-sandbox.sym | 4 ++ 5 files changed, 91 insertions(+)
Reviewed-by: Daniel P. Berrange <berrange@redhat.com>
if the commit message is fixed
Regards, Daniel

On Wed, Dec 06, 2017 at 01:56:23PM +0100, Cedric Bosdonnat wrote:
On Tue, 2017-12-05 at 16:52 +0000, Daniel P. Berrange wrote:
On Tue, Dec 05, 2017 at 10:53:17AM +0100, Cédric Bosdonnat wrote:
libvirt-sandbox-init-common is expecting -d and -v parameters to set it in debug or verbose mode... but those will never be passed by the launcher program.
Hmm, libvirt-sandbox-init-{qemu,lxc} both know how to pass -d to init-common. If either of the virt specific init programs are in debug mode (as indicated by the word "debug" in kernel command line or equiv), then they pass -d to init-common.
Hum, then I should surely revise that commit to remove the debug-related code. Instead I should change the init-{lxc,qemu} to actually pass the debug flag to init-common.
You're right that there's no way to pass -v though.
Also, it might be useful to run init-common in debug mode without the virt specific init being in debug mode, to avoid drowning in debug.
Hum, then keeping the debug-related flag I introduced could help, but then I'll need to rename it to make it more explicit. Or maybe I should change the debug parameter to take a level value: 0: all (default) 1: only the init-common
I'm not a fan of adding args to --debug flags. I think what you have here already is good enough, just with commit message changed to be more accurate
In which case the user-exposed options should be expanded to use these.
I'll push the other commits, but not this one.
-- Cedric
Writing the core.debug and core.verbose parameters in the sandbox configuration makes those values actually usable from the init. --- bin/virt-sandbox.c | 3 ++ libvirt-sandbox/libvirt-sandbox-config.c | 75 +++++++++++++++++++++++++++ libvirt-sandbox/libvirt-sandbox-config.h | 6 +++ libvirt-sandbox/libvirt-sandbox-init-common.c | 3 ++ libvirt-sandbox/libvirt-sandbox.sym | 4 ++ 5 files changed, 91 insertions(+)
Reviewed-by: Daniel P. Berrange <berrange@redhat.com>
if the commit message is fixed
Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

When running virt-sandbox as a user with host-bind mount, the user can't write in the mounted folder. If run as root, use passthrough security mode, otherwise use squashed one to fix this. --- libvirt-sandbox/libvirt-sandbox-builder-machine.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libvirt-sandbox/libvirt-sandbox-builder-machine.c b/libvirt-sandbox/libvirt-sandbox-builder-machine.c index 7204f71..b6f2218 100644 --- a/libvirt-sandbox/libvirt-sandbox-builder-machine.c +++ b/libvirt-sandbox/libvirt-sandbox-builder-machine.c @@ -589,7 +589,10 @@ static gboolean gvir_sandbox_builder_machine_construct_devices(GVirSandboxBuilde fs = gvir_config_domain_filesys_new(); gvir_config_domain_filesys_set_type(fs, GVIR_CONFIG_DOMAIN_FILESYS_MOUNT); - gvir_config_domain_filesys_set_access_type(fs, GVIR_CONFIG_DOMAIN_FILESYS_ACCESS_PASSTHROUGH); + if (getuid() == 0) + gvir_config_domain_filesys_set_access_type(fs, GVIR_CONFIG_DOMAIN_FILESYS_ACCESS_PASSTHROUGH); + else + gvir_config_domain_filesys_set_access_type(fs, GVIR_CONFIG_DOMAIN_FILESYS_ACCESS_SQUASH); gvir_config_domain_filesys_set_source(fs, gvir_sandbox_config_mount_file_get_source(mfile)); gvir_config_domain_filesys_set_target(fs, target); -- 2.15.1

On Tue, Dec 05, 2017 at 10:53:18AM +0100, Cédric Bosdonnat wrote:
When running virt-sandbox as a user with host-bind mount, the user can't write in the mounted folder. If run as root, use passthrough security mode, otherwise use squashed one to fix this. --- libvirt-sandbox/libvirt-sandbox-builder-machine.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
Reviewed-by: Daniel P. Berrange <berrange@redhat.com> Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

--- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 83831bb..c4e2217 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,5 @@ bin/virt-sandbox bin/virt-sandbox-service-util build/ bin/*.1 +*.log +*.trs -- 2.15.1

On Tue, Dec 05, 2017 at 10:53:19AM +0100, Cédric Bosdonnat wrote:
--- .gitignore | 2 ++ 1 file changed, 2 insertions(+)
Reviewed-by: Daniel P. Berrange <berrange@redhat.com> Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

Since commit 68406aff8 ConfigMountHostImage needs a format parameter, virt-sandbox-service needs to fit the new API. --- bin/virt-sandbox-service | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/virt-sandbox-service b/bin/virt-sandbox-service index 45f4517..c34c6f3 100755 --- a/bin/virt-sandbox-service +++ b/bin/virt-sandbox-service @@ -20,6 +20,8 @@ # import gi +gi.require_version('LibvirtGConfig', '1.0') +from gi.repository import LibvirtGConfig gi.require_version('LibvirtGObject', '1.0') from gi.repository import LibvirtGObject gi.require_version('LibvirtSandbox', '1.0') @@ -223,7 +225,8 @@ class Container: def gen_filesystems(self): if self.use_image: self.image = self.DEFAULT_IMAGE % self.get_name() - mount = LibvirtSandbox.ConfigMountHostImage.new(self.image, self.dest) + mount = LibvirtSandbox.ConfigMountHostImage.new(self.image, self.dest, + LibvirtGConfig.DomainDiskFormat.RAW) self.config.add_mount(mount) def fix_stat(self, f): -- 2.15.1

On Tue, Dec 05, 2017 at 10:53:20AM +0100, Cédric Bosdonnat wrote:
Since commit 68406aff8 ConfigMountHostImage needs a format parameter, virt-sandbox-service needs to fit the new API. --- bin/virt-sandbox-service | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
Reviewed-by: Daniel P. Berrange <berrange@redhat.com> Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

Python2 is going to die soon, convert to python3. --- bin/virt-sandbox-service | 68 ++++++++++++++++++--------------- libvirt-sandbox/image/cli.py | 18 ++++----- libvirt-sandbox/image/sources/docker.py | 41 ++++++++++---------- libvirt-sandbox/image/template.py | 8 ++-- 4 files changed, 71 insertions(+), 64 deletions(-) diff --git a/bin/virt-sandbox-service b/bin/virt-sandbox-service index c34c6f3..e78defb 100755 --- a/bin/virt-sandbox-service +++ b/bin/virt-sandbox-service @@ -30,7 +30,6 @@ from gi.repository import GLib import gi import re import os, sys, shutil, errno, stat -import exceptions import rpm from subprocess import Popen, PIPE, STDOUT import gettext @@ -49,7 +48,6 @@ gettext.textdomain("libvirt-sandbox") try: gettext.install("libvirt-sandbox", localedir="/usr/share/locale", - unicode=False, codeset = 'utf-8') except IOError: import __builtin__ @@ -235,7 +233,7 @@ class Container: path = "%s%s" % (self.dest, f) os.chown(path, s.st_uid, s.st_gid) os.chmod(path, s.st_mode) - except OSError, e: + except OSError as e: if not e.errno == errno.ENOENT: raise @@ -253,7 +251,7 @@ class Container: try: path = "%s%s" % (self.dest, d) os.makedirs(path) - except OSError, e: + except OSError as e: if not e.errno == errno.EEXIST: raise @@ -263,7 +261,7 @@ class Container: path = "%s%s" % (self.dest, f) fd=open(path, "w") fd.close() - except OSError, e: + except OSError as e: if not e.errno == errno.EEXIST: raise @@ -404,10 +402,10 @@ class GenericContainer(Container): def create(self): try: self.create_generic() - except Exception, e: + except Exception as e: try: self.delete() - except Exception, e2: + except Exception as e2: pass raise e @@ -418,6 +416,18 @@ class GenericContainer(Container): def is_template_unit(unit): return '@' in unit +# Python 2 / 3 compability helpers +def get_next(obj): + if hasattr(obj, 'next'): + return obj.next() + else: + return next(obj) + +def string(obj): + if isinstance(obj, bytes): + return str(obj, encoding='utf-8') + return obj + class SystemdContainer(Container): IGNORE_DIRS = [ "/var/run/", "/etc/logrotate.d/", "/etc/pam.d" ] DEFAULT_DIRS = [ "/etc", "/var" ] @@ -581,8 +591,8 @@ WantedBy=multi-user.target def get_rpm_for_unit(self, unitfile): mi = self.ts.dbMatch(rpm.RPMTAG_BASENAMES, unitfile) try: - h = mi.next(); - except exceptions.StopIteration: + h = get_next(mi); + except StopIteration: return None return h['name'] @@ -590,8 +600,8 @@ WantedBy=multi-user.target def extract_rpm(self, rpm_name): mi = self.ts.dbMatch('name', rpm_name) try: - h = mi.next(); - except exceptions.StopIteration: + h = get_next(mi); + except StopIteration: raise ValueError([_("Cannot find package named %s") % rpm_name]) for fentry in h.fiFromHeader(): @@ -602,16 +612,16 @@ WantedBy=multi-user.target if os.path.isfile(fname): self.add_file(fname) - srcrpm = h[rpm.RPMTAG_SOURCERPM] + srcrpm = string(h[rpm.RPMTAG_SOURCERPM]) srcrpmbits = self.split_filename(srcrpm) - if srcrpmbits[0] == h[rpm.RPMTAG_NAME]: + if srcrpmbits[0] == string(h[rpm.RPMTAG_NAME]): return mi = self.ts.dbMatch(rpm.RPMTAG_NAME, srcrpmbits[0]) try: - h = mi.next(); - except exceptions.StopIteration: + h = get_next(mi); + except StopIteration: raise ValueError([_("Cannot find base package %s") % srcrpmbits[0]]) for fentry in h.fiFromHeader(): @@ -771,7 +781,7 @@ PrivateNetwork=false fd.write("[Unit]\n") fd.write("Description=Sandbox multi-user target\n") fd.close() - except OSError, e: + except OSError as e: if not e.errno == errno.EEXIST: raise @@ -789,7 +799,7 @@ PrivateNetwork=false jpath = "/var/log/journal/" + uuid if os.path.lexists(jpath): os.remove(jpath) - except Exception, e: + except Exception as e: sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) sys.stderr.flush() @@ -825,10 +835,10 @@ PrivateNetwork=false try: self.create_systemd() - except Exception, e: + except Exception as e: try: self.delete() - except Exception, e2: + except Exception as e2: sys.stderr.write("Cleanup failed: %s\n" % str(e2)) raise @@ -923,10 +933,10 @@ def connect(args): execute(args) return - print """\ + print ("""\ Connected to %s. Type 'Ctrl + ]' to detach from the console. -""" % ( args.name ) +""" % ( args.name )) os.execl("/usr/libexec/virt-sandbox-service-util", "virt-sandbox-service-util", "-c", args.uri, @@ -1014,7 +1024,7 @@ def clone(args): newcontainer.set_security(args.security) newcontainer.set_security_label() newcontainer.save_config() - except Exception, e: + except Exception as e: if newcontainer is not None: newcontainer.delete() raise @@ -1296,23 +1306,21 @@ if __name__ == '__main__': sys.exit(1) args.func(args) sys.exit(0) - except KeyboardInterrupt, e: + except KeyboardInterrupt as e: sys.exit(0) - except ValueError, e: - for line in e: - for l in line: - sys.stderr.write("%s: %s\n" % (sys.argv[0], l)) + except ValueError as e: + sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) sys.stderr.flush() sys.exit(1) - except IOError, e: + except IOError as e: sys.stderr.write("%s: %s: %s\n" % (sys.argv[0], e.filename, e.strerror)) sys.stderr.flush() sys.exit(1) - except OSError, e: + except OSError as e: sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) sys.stderr.flush() sys.exit(1) - except GLib.GError, e: + except GLib.GError as e: sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) sys.stderr.flush() sys.exit(1) diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py index d5e624c..fa3cace 100644 --- a/libvirt-sandbox/image/cli.py +++ b/libvirt-sandbox/image/cli.py @@ -30,7 +30,6 @@ import os.path import re import shutil import sys -import urllib2 import subprocess import random import string @@ -52,7 +51,6 @@ gettext.textdomain("libvirt-sandbox") try: gettext.install("libvirt-sandbox", localedir="/usr/share/locale", - unicode=False, codeset = 'utf-8') except IOError: import __builtin__ @@ -149,7 +147,7 @@ def list_cached(args): tmpls.extend(template.Template.get_all(source, "%s/%s" % (args.template_dir, source))) for tmpl in tmpls: - print tmpl + print (tmpl) def requires_template(parser): parser.add_argument("template", @@ -265,20 +263,20 @@ def main(): try: args.func(args) sys.exit(0) - except KeyboardInterrupt, e: + except KeyboardInterrupt as e: sys.exit(0) - except ValueError, e: + except ValueError as e: sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) sys.stderr.flush() sys.exit(1) - except IOError, e: - sys.stderr.write("%s: %s: %s\n" % (sys.argv[0], e.filename, e.reason)) + except IOError as e: + sys.stderr.write("%s: %s\n" % (sys.argv[0], e.filename)) sys.stderr.flush() sys.exit(1) - except OSError, e: + except OSError as e: sys.stderr.write("%s: %s\n" % (sys.argv[0], e)) sys.stderr.flush() sys.exit(1) - except Exception, e: - print e.message + except Exception as e: + print (e) sys.exit(1) diff --git a/libvirt-sandbox/image/sources/docker.py b/libvirt-sandbox/image/sources/docker.py index 6ca086c..e979054 100755 --- a/libvirt-sandbox/image/sources/docker.py +++ b/libvirt-sandbox/image/sources/docker.py @@ -21,14 +21,15 @@ # Author: Eren Yagdiran <erenyagdiran@gmail.com> # -import urllib2 import sys import json import traceback import os import subprocess import shutil -import urlparse +import urllib.error +import urllib.parse +import urllib.request import hashlib from abc import ABCMeta, abstractmethod import copy @@ -133,7 +134,7 @@ class DockerAuthBasic(DockerAuth): req.add_header("X-Docker-Token", "true") def process_res(self, res): - self.token = res.info().getheader('X-Docker-Token') + self.token = res.info().get('X-Docker-Token') def process_err(self, err): return False @@ -194,10 +195,10 @@ class DockerAuthBearer(DockerAuth): if params != "": url = url + "?" + params - req = urllib2.Request(url=url) + req = urllib.request.Request(url=url) req.add_header("Accept", "application/json") - res = urllib2.urlopen(req) + res = urllib.request.urlopen(req) data = json.loads(res.read()) self.token = data["token"] return True @@ -207,7 +208,7 @@ class DockerRegistry(): def __init__(self, uri_base): - self.uri_base = list(urlparse.urlparse(uri_base)) + self.uri_base = list(urllib.parse.urlparse(uri_base)) self.auth_handler = DockerAuthNop() def set_auth_handler(self, auth_handler): @@ -216,8 +217,8 @@ class DockerRegistry(): def supports_v2(self): try: (data, res) = self.get_json("/v2/") - ver = res.info().getheader("Docker-Distribution-Api-Version") - except urllib2.HTTPError as e: + ver = res.info().get("Docker-Distribution-Api-Version") + except urllib.error.HTTPError as e: ver = e.headers.get("Docker-Distribution-Api-Version", None) if ver is None: @@ -243,17 +244,17 @@ class DockerRegistry(): else: server = "%s:%s" % (hostname, port) - url = urlparse.urlunparse((protocol, server, "", None, None, None)) + url = urllib.parse.urlunparse((protocol, server, "", None, None, None)) return cls(url) def get_url(self, path, headers=None): url_bits = copy.copy(self.uri_base) url_bits[2] = path - url = urlparse.urlunparse(url_bits) + url = urllib.parse.urlunparse(url_bits) debug("Fetching %s..." % url) - req = urllib2.Request(url=url) + req = urllib.request.Request(url=url) if headers is not None: for h in headers.keys(): @@ -262,16 +263,16 @@ class DockerRegistry(): self.auth_handler.prepare_req(req) try: - res = urllib2.urlopen(req) + res = urllib.request.urlopen(req) self.auth_handler.process_res(res) return res - except urllib2.HTTPError as e: + except urllib.error.HTTPError as e: if e.code == 401: retry = self.auth_handler.process_err(e) if retry: debug("Re-Fetching %s..." % url) self.auth_handler.prepare_req(req) - res = urllib2.urlopen(req) + res = urllib.request.urlopen(req) self.auth_handler.process_res(res) return res else: @@ -284,7 +285,7 @@ class DockerRegistry(): try: res = self.get_url(path) - datalen = res.info().getheader("Content-Length") + datalen = res.info().get("Content-Length") if datalen is not None: datalen = int(datalen) @@ -296,7 +297,7 @@ class DockerRegistry(): patternIndex = 0 donelen = 0 - with open(dest, "w") as f: + with open(dest, "wb") as f: while 1: buf = res.read(1024*64) if not buf: @@ -321,7 +322,7 @@ class DockerRegistry(): raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum)) debug("OK\n") return res - except Exception, e: + except Exception as e: debug("FAIL %s\n" % str(e)) raise @@ -333,7 +334,7 @@ class DockerRegistry(): data = json.loads(res.read()) debug("OK\n") return (data, res) - except Exception, e: + except Exception as e: debug("FAIL %s\n" % str(e)) raise @@ -426,10 +427,10 @@ class DockerSource(base.Source): (data, res) = registry.get_json("/v1/repositories/%s/%s/images" % ( image.repo, image.name, )) - except urllib2.HTTPError, e: + except urllib.error.HTTPError as e: raise ValueError(["Image '%s' does not exist" % template]) - registryendpoint = res.info().getheader('X-Docker-Endpoints') + registryendpoint = res.info().get('X-Docker-Endpoints') if basicauth.token is not None: registry.set_auth_handler(DockerAuthToken(basicauth.token)) diff --git a/libvirt-sandbox/image/template.py b/libvirt-sandbox/image/template.py index 79dc33d..ab2ea29 100644 --- a/libvirt-sandbox/image/template.py +++ b/libvirt-sandbox/image/template.py @@ -19,7 +19,7 @@ # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # -import urlparse +import urllib.parse import importlib import re @@ -71,7 +71,7 @@ class Template(object): classimpl = getattr(mod, classname) return classimpl() except Exception as e: - print e + print (e) raise Exception("Invalid source: '%s'" % source) def get_source_impl(self): @@ -101,12 +101,12 @@ class Template(object): netloc = None query = "&".join([key + "=" + self.params[key] for key in self.params.keys()]) - ret = urlparse.urlunparse((scheme, netloc, self.path, None, query, None)) + ret = urllib.parse.urlunparse((scheme, netloc, self.path, None, query, None)) return ret @classmethod def from_uri(klass, uri): - o = urlparse.urlparse(uri) + o = urllib.parse.urlparse(uri) idx = o.scheme.find("+") if idx == -1: -- 2.15.1

On Tue, Dec 05, 2017 at 10:53:21AM +0100, Cédric Bosdonnat wrote:
Python2 is going to die soon, convert to python3.
I'm unclear whether this change drops py2 support, or whether it makes it work with py2+3 in parallel. The commit message suggests py3 only, but then this:
@@ -418,6 +416,18 @@ class GenericContainer(Container): def is_template_unit(unit): return '@' in unit
+# Python 2 / 3 compability helpers +def get_next(obj): + if hasattr(obj, 'next'): + return obj.next() + else: + return next(obj) + +def string(obj): + if isinstance(obj, bytes): + return str(obj, encoding='utf-8') + return obj
suggests py2+3 in parallel I don't mind being py3 only. Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

On Tue, 2017-12-05 at 16:57 +0000, Daniel P. Berrange wrote:
On Tue, Dec 05, 2017 at 10:53:21AM +0100, Cédric Bosdonnat wrote:
Python2 is going to die soon, convert to python3.
I'm unclear whether this change drops py2 support, or whether it makes it work with py2+3 in parallel.
The commit message suggests py3 only, but then this:
@@ -418,6 +416,18 @@ class GenericContainer(Container): def is_template_unit(unit): return '@' in unit
+# Python 2 / 3 compability helpers +def get_next(obj): + if hasattr(obj, 'next'): + return obj.next() + else: + return next(obj) + +def string(obj): + if isinstance(obj, bytes): + return str(obj, encoding='utf-8') + return obj
suggests py2+3 in parallel
I don't mind being py3 only.
Oops, that one is a left over. I'll remove it and drop python2. And at least the commit message will be in sync with the code -- Cedric

On Wed, Dec 06, 2017 at 01:50:56PM +0100, Cedric Bosdonnat wrote:
On Tue, 2017-12-05 at 16:57 +0000, Daniel P. Berrange wrote:
On Tue, Dec 05, 2017 at 10:53:21AM +0100, Cédric Bosdonnat wrote:
Python2 is going to die soon, convert to python3.
I'm unclear whether this change drops py2 support, or whether it makes it work with py2+3 in parallel.
The commit message suggests py3 only, but then this:
@@ -418,6 +416,18 @@ class GenericContainer(Container): def is_template_unit(unit): return '@' in unit
+# Python 2 / 3 compability helpers +def get_next(obj): + if hasattr(obj, 'next'): + return obj.next() + else: + return next(obj) + +def string(obj): + if isinstance(obj, bytes): + return str(obj, encoding='utf-8') + return obj
suggests py2+3 in parallel
I don't mind being py3 only.
Oops, that one is a left over. I'll remove it and drop python2. And at least the commit message will be in sync with the code
Ok, ack with that dropped Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

This is particularly useful on operating systems that don't ship Python as part of the base system (eg. FreeBSD) while still working just as well as it did before on Linux. --- bin/virt-sandbox-image | 2 +- bin/virt-sandbox-service | 2 +- examples/demo.py | 2 +- examples/shell.py | 2 +- examples/virt-sandbox-mkinitrd.py | 2 +- examples/virt-sandbox.py | 2 +- libvirt-sandbox/image/cli.py | 2 +- libvirt-sandbox/image/sources/base.py | 1 - libvirt-sandbox/image/sources/docker.py | 1 - libvirt-sandbox/image/sources/virtbuilder.py | 1 - 10 files changed, 7 insertions(+), 10 deletions(-) diff --git a/bin/virt-sandbox-image b/bin/virt-sandbox-image index 7e0d76b..61346ef 100755 --- a/bin/virt-sandbox-image +++ b/bin/virt-sandbox-image @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- from libvirt_sandbox.image import cli diff --git a/bin/virt-sandbox-service b/bin/virt-sandbox-service index e78defb..b06f304 100755 --- a/bin/virt-sandbox-service +++ b/bin/virt-sandbox-service @@ -1,4 +1,4 @@ -#!/usr/bin/python -Es +#!/usr/bin/env python3 # # Authors: Dan Walsh <dwalsh@redhat.com> # diff --git a/examples/demo.py b/examples/demo.py index 6252194..c61b091 100755 --- a/examples/demo.py +++ b/examples/demo.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 from gi.repository import LibvirtGObject from gi.repository import LibvirtSandbox diff --git a/examples/shell.py b/examples/shell.py index 317d13e..86b2159 100755 --- a/examples/shell.py +++ b/examples/shell.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 from gi.repository import LibvirtGObject from gi.repository import LibvirtSandbox diff --git a/examples/virt-sandbox-mkinitrd.py b/examples/virt-sandbox-mkinitrd.py index 76df925..74d1b00 100755 --- a/examples/virt-sandbox-mkinitrd.py +++ b/examples/virt-sandbox-mkinitrd.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 from gi.repository import LibvirtGObject from gi.repository import LibvirtSandbox diff --git a/examples/virt-sandbox.py b/examples/virt-sandbox.py index 922ad00..e4ed6e7 100755 --- a/examples/virt-sandbox.py +++ b/examples/virt-sandbox.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 from gi.repository import LibvirtGObject from gi.repository import LibvirtSandbox diff --git a/libvirt-sandbox/image/cli.py b/libvirt-sandbox/image/cli.py index fa3cace..490c5e0 100644 --- a/libvirt-sandbox/image/cli.py +++ b/libvirt-sandbox/image/cli.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -Es +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Authors: Daniel P. Berrange <berrange@redhat.com> # Eren Yagdiran <erenyagdiran@gmail.com> diff --git a/libvirt-sandbox/image/sources/base.py b/libvirt-sandbox/image/sources/base.py index e4e4e41..0fc9243 100644 --- a/libvirt-sandbox/image/sources/base.py +++ b/libvirt-sandbox/image/sources/base.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. diff --git a/libvirt-sandbox/image/sources/docker.py b/libvirt-sandbox/image/sources/docker.py index e979054..eaf41fc 100755 --- a/libvirt-sandbox/image/sources/docker.py +++ b/libvirt-sandbox/image/sources/docker.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # -*- coding: utf-8 -*- # # Copyright (C) 2015 Universitat Politècnica de Catalunya. diff --git a/libvirt-sandbox/image/sources/virtbuilder.py b/libvirt-sandbox/image/sources/virtbuilder.py index 6bd9695..1b32083 100755 --- a/libvirt-sandbox/image/sources/virtbuilder.py +++ b/libvirt-sandbox/image/sources/virtbuilder.py @@ -1,4 +1,3 @@ -#!/usr/bin/python # # Copyright (C) 2015 SUSE LLC # -- 2.15.1

On Tue, Dec 05, 2017 at 10:53:22AM +0100, Cédric Bosdonnat wrote:
This is particularly useful on operating systems that don't ship Python as part of the base system (eg. FreeBSD) while still working just as well as it did before on Linux. --- bin/virt-sandbox-image | 2 +- bin/virt-sandbox-service | 2 +- examples/demo.py | 2 +- examples/shell.py | 2 +- examples/virt-sandbox-mkinitrd.py | 2 +- examples/virt-sandbox.py | 2 +- libvirt-sandbox/image/cli.py | 2 +- libvirt-sandbox/image/sources/base.py | 1 - libvirt-sandbox/image/sources/docker.py | 1 - libvirt-sandbox/image/sources/virtbuilder.py | 1 - 10 files changed, 7 insertions(+), 10 deletions(-)
Reviewed-by: Daniel P. Berrange <berrange@redhat.com> Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
participants (3)
-
Cedric Bosdonnat
-
Cédric Bosdonnat
-
Daniel P. Berrange