[libvirt] [PATCH] lxc: Bind mount container TTYs
Instead of creating symlinks, bind mount the devices to /dev/pts/XY. Using bind mounts it is no longer needed to add pts devices to files like /dev/securetty. Signed-off-by: Richard Weinberger <richard@nod.at> --- src/lxc/lxc_container.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/lxc/lxc_container.c b/src/lxc/lxc_container.c index 7d531e2..ea76370 100644 --- a/src/lxc/lxc_container.c +++ b/src/lxc/lxc_container.c @@ -1141,6 +1141,20 @@ static int lxcContainerMountFSDevPTS(virDomainDefPtr def, return ret; } +static int lxcContainerBindMountDevice(const char *src, const char *dst) +{ + if (virFileTouch(dst, 0666) < 0) + return -1; + + if (mount(src, dst, "none", MS_BIND, NULL) < 0) { + virReportSystemError(errno, _("Failed to bind %s on to %s"), src, + dst); + return -1; + } + + return 0; +} + static int lxcContainerSetupDevices(char **ttyPaths, size_t nttyPaths) { size_t i; @@ -1164,34 +1178,24 @@ static int lxcContainerSetupDevices(char **ttyPaths, size_t nttyPaths) } /* We have private devpts capability, so bind that */ - if (virFileTouch("/dev/ptmx", 0666) < 0) + if (lxcContainerBindMountDevice("/dev/pts/ptmx", "/dev/ptmx") < 0) return -1; - if (mount("/dev/pts/ptmx", "/dev/ptmx", "ptmx", MS_BIND, NULL) < 0) { - virReportSystemError(errno, "%s", - _("Failed to bind /dev/pts/ptmx on to /dev/ptmx")); - return -1; - } - for (i = 0; i < nttyPaths; i++) { char *tty; if (virAsprintf(&tty, "/dev/tty%zu", i+1) < 0) return -1; - if (symlink(ttyPaths[i], tty) < 0) { - virReportSystemError(errno, - _("Failed to symlink %s to %s"), - ttyPaths[i], tty); - VIR_FREE(tty); + + if (lxcContainerBindMountDevice(ttyPaths[i], tty) < 0) { return -1; + VIR_FREE(tty); } + VIR_FREE(tty); + if (i == 0 && - symlink(ttyPaths[i], "/dev/console") < 0) { - virReportSystemError(errno, - _("Failed to symlink %s to /dev/console"), - ttyPaths[i]); + lxcContainerBindMountDevice(ttyPaths[i], "/dev/console") < 0) return -1; - } } return 0; } -- 2.4.2
On Tue, Jun 23, 2015 at 04:38:57PM +0200, Richard Weinberger wrote:
Instead of creating symlinks, bind mount the devices to /dev/pts/XY. Using bind mounts it is no longer needed to add pts devices to files like /dev/securetty.
I guess you meant /etc/securetty. This patch makes sense, but if I start a container that I couldn't login as a root into (because of securetty), it still doesn't help, I still can't login. Moreover, if I stop it and start it few times and restart the daemon (I'm not sure whether that's needed, it's just that I had to switch between gdb and non-gdb daemons and it happened only sometimes), I get this: error: internal error: guest failed to start: unexpected exit status 125 The error in log is: libvirt: error : failed to setup stdout file handle: Bad file descriptor I briefly looked at it and *cmd->outfdptr has the value of 247083264 which is nowhere in the output of lsof for that process. I know that it doesn't sounds even remotely related, but without this patch that doesn't happen. Maybe it just uncovers some error rotting there for a long time...
Signed-off-by: Richard Weinberger <richard@nod.at> --- src/lxc/lxc_container.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-)
diff --git a/src/lxc/lxc_container.c b/src/lxc/lxc_container.c index 7d531e2..ea76370 100644 --- a/src/lxc/lxc_container.c +++ b/src/lxc/lxc_container.c @@ -1141,6 +1141,20 @@ static int lxcContainerMountFSDevPTS(virDomainDefPtr def, return ret; }
+static int lxcContainerBindMountDevice(const char *src, const char *dst) +{ + if (virFileTouch(dst, 0666) < 0) + return -1; + + if (mount(src, dst, "none", MS_BIND, NULL) < 0) { + virReportSystemError(errno, _("Failed to bind %s on to %s"), src, + dst); + return -1; + } + + return 0; +} + static int lxcContainerSetupDevices(char **ttyPaths, size_t nttyPaths) { size_t i; @@ -1164,34 +1178,24 @@ static int lxcContainerSetupDevices(char **ttyPaths, size_t nttyPaths) }
/* We have private devpts capability, so bind that */ - if (virFileTouch("/dev/ptmx", 0666) < 0) + if (lxcContainerBindMountDevice("/dev/pts/ptmx", "/dev/ptmx") < 0) return -1;
- if (mount("/dev/pts/ptmx", "/dev/ptmx", "ptmx", MS_BIND, NULL) < 0) { - virReportSystemError(errno, "%s", - _("Failed to bind /dev/pts/ptmx on to /dev/ptmx")); - return -1; - } - for (i = 0; i < nttyPaths; i++) { char *tty; if (virAsprintf(&tty, "/dev/tty%zu", i+1) < 0) return -1; - if (symlink(ttyPaths[i], tty) < 0) { - virReportSystemError(errno, - _("Failed to symlink %s to %s"), - ttyPaths[i], tty); - VIR_FREE(tty); + + if (lxcContainerBindMountDevice(ttyPaths[i], tty) < 0) { return -1; + VIR_FREE(tty); } + VIR_FREE(tty); + if (i == 0 && - symlink(ttyPaths[i], "/dev/console") < 0) { - virReportSystemError(errno, - _("Failed to symlink %s to /dev/console"), - ttyPaths[i]); + lxcContainerBindMountDevice(ttyPaths[i], "/dev/console") < 0) return -1; - } } return 0; } -- 2.4.2
-- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
Am 26.06.2015 um 15:09 schrieb Martin Kletzander:
On Tue, Jun 23, 2015 at 04:38:57PM +0200, Richard Weinberger wrote:
Instead of creating symlinks, bind mount the devices to /dev/pts/XY. Using bind mounts it is no longer needed to add pts devices to files like /dev/securetty.
I guess you meant /etc/securetty.
Lol, yes. :-)
This patch makes sense, but if I start a container that I couldn't login as a root into (because of securetty), it still doesn't help, I still can't login. Moreover, if I stop it and start it few times and restart the daemon (I'm not sure whether that's needed, it's just that I had to switch between gdb and non-gdb daemons and it happened only sometimes), I get this:
error: internal error: guest failed to start: unexpected exit status 125
The error in log is:
libvirt: error : failed to setup stdout file handle: Bad file descriptor
I briefly looked at it and *cmd->outfdptr has the value of 247083264 which is nowhere in the output of lsof for that process. I know that it doesn't sounds even remotely related, but without this patch that doesn't happen. Maybe it just uncovers some error rotting there for a long time...
Hmm, very strange. What guest container are you using? I tried with a Debian jessi and had user namespace enabled. Thanks, //richard
On Sun, Jun 28, 2015 at 09:46:43AM +0200, Richard Weinberger wrote:
Am 26.06.2015 um 15:09 schrieb Martin Kletzander:
On Tue, Jun 23, 2015 at 04:38:57PM +0200, Richard Weinberger wrote:
Instead of creating symlinks, bind mount the devices to /dev/pts/XY. Using bind mounts it is no longer needed to add pts devices to files like /dev/securetty.
I guess you meant /etc/securetty.
Lol, yes. :-)
This patch makes sense, but if I start a container that I couldn't login as a root into (because of securetty), it still doesn't help, I still can't login. Moreover, if I stop it and start it few times and restart the daemon (I'm not sure whether that's needed, it's just that I had to switch between gdb and non-gdb daemons and it happened only sometimes), I get this:
error: internal error: guest failed to start: unexpected exit status 125
The error in log is:
libvirt: error : failed to setup stdout file handle: Bad file descriptor
I briefly looked at it and *cmd->outfdptr has the value of 247083264 which is nowhere in the output of lsof for that process. I know that it doesn't sounds even remotely related, but without this patch that doesn't happen. Maybe it just uncovers some error rotting there for a long time...
Hmm, very strange. What guest container are you using? I tried with a Debian jessi and had user namespace enabled.
Sorry for the late reply. I used simple one. Only gentoo's stage 3 unpacked into a directory, no special settings used for it. Removing /etc/securetty works for me. I'll give it another try, but probably after the freeze. If anyone else wants to review this, don't get stopped by the problems I'm having!
Thanks, //richard
Am 30.06.2015 um 19:12 schrieb Martin Kletzander:
Hmm, very strange. What guest container are you using? I tried with a Debian jessi and had user namespace enabled.
Sorry for the late reply. I used simple one. Only gentoo's stage 3 unpacked into a directory, no special settings used for it. Removing /etc/securetty works for me. I'll give it another try, but probably after the freeze. If anyone else wants to review this, don't get stopped by the problems I'm having!
Hmm, just gave gentoo a try, worked perfectly fine. Can you share your xml? This is mine: <domain type='lxc'> <name>gentoo</name> <memory>524288</memory> <os> <type>exe</type> <init>/sbin/init</init> </os> <idmap> <uid start='0' target='100000' count='998'/> <gid start='0' target='100000' count='998'/> <uid start='65533' target='100998' count='2'/> <gid start='65533' target='100998' count='2'/> </idmap> <devices> <console type='pty'/> <filesystem type='mount'> <source dir='/srv/container/gentoo/'/> <target dir='/'/> </filesystem> <interface type='bridge'> <source bridge='br0'/> <mac address='52:54:00:44:55:66'/> </interface> </devices> </domain> Thanks, //richard
On Tue, Jun 30, 2015 at 07:54:25PM +0200, Richard Weinberger wrote:
Am 30.06.2015 um 19:12 schrieb Martin Kletzander:
Hmm, very strange. What guest container are you using? I tried with a Debian jessi and had user namespace enabled.
Sorry for the late reply. I used simple one. Only gentoo's stage 3 unpacked into a directory, no special settings used for it. Removing /etc/securetty works for me. I'll give it another try, but probably after the freeze. If anyone else wants to review this, don't get stopped by the problems I'm having!
Hmm, just gave gentoo a try, worked perfectly fine.
I tried with latest master with and without your patch. Wtih your patch I got to the problem exactly once even though I tried multiple times. And even though it didn't happen to me at all without your patch, I'm thinking it's just some weird rare race and it's not related to what you've sent. That just wouldn't make sense to me. I also suspected the problem being me starting with --console parameter, but trying with and without that didn't help isolate the problem either. Anyway, that patch still doesn't help me get rid of /etc/securetty. The output of 'tty' is still /dev/pts/0 and unless I remove /etc/securetty it doesn't start. What is the output of 'tty' and what ttys do you have in /etc/securetty in your container?
Can you share your xml?
Sure, mine is almost same as yours except the user namespace isolation. <domain type='lxc'> <name>gentoo</name> <uuid>9de0da50-bddd-40e5-ba4a-24c2ed8fca05</uuid> <memory unit='KiB'>1048576</memory> <currentMemory unit='KiB'>1048576</currentMemory> <vcpu placement='auto'>1</vcpu> <numatune> <memory mode='strict' placement='auto'/> </numatune> <resource> <partition>/machine</partition> </resource> <os> <type arch='x86_64'>exe</type> <init>/sbin/init</init> </os> <features> <acpi/> </features> <clock offset='utc'/> <on_poweroff>destroy</on_poweroff> <on_reboot>restart</on_reboot> <on_crash>destroy</on_crash> <devices> <emulator>/usr/libexec/libvirt_lxc</emulator> <filesystem type='mount' accessmode='passthrough'> <source dir='/mnt/lxc_gentoo'/> <target dir='/'/> </filesystem> <interface type='bridge'> <mac address='52:54:00:ae:8b:47'/> <source bridge='virbr0'/> <target dev='viftestmaster'/> <guest dev='eth0'/> </interface> <console type='pty'> <target type='lxc' port='0'/> </console> </devices> </domain>
This is mine: <domain type='lxc'> <name>gentoo</name> <memory>524288</memory> <os> <type>exe</type> <init>/sbin/init</init> </os> <idmap> <uid start='0' target='100000' count='998'/> <gid start='0' target='100000' count='998'/> <uid start='65533' target='100998' count='2'/> <gid start='65533' target='100998' count='2'/> </idmap> <devices> <console type='pty'/> <filesystem type='mount'> <source dir='/srv/container/gentoo/'/> <target dir='/'/> </filesystem> <interface type='bridge'> <source bridge='br0'/> <mac address='52:54:00:44:55:66'/> </interface> </devices> </domain>
Thanks, //richard
On Wed, Jul 01, 2015 at 11:40:38AM +0200, Martin Kletzander wrote:
On Tue, Jun 30, 2015 at 07:54:25PM +0200, Richard Weinberger wrote:
Am 30.06.2015 um 19:12 schrieb Martin Kletzander:
Hmm, very strange. What guest container are you using? I tried with a Debian jessi and had user namespace enabled.
Sorry for the late reply. I used simple one. Only gentoo's stage 3 unpacked into a directory, no special settings used for it. Removing /etc/securetty works for me. I'll give it another try, but probably after the freeze. If anyone else wants to review this, don't get stopped by the problems I'm having!
Hmm, just gave gentoo a try, worked perfectly fine.
I tried with latest master with and without your patch. Wtih your patch I got to the problem exactly once even though I tried multiple times. And even though it didn't happen to me at all without your patch, I'm thinking it's just some weird rare race and it's not related to what you've sent. That just wouldn't make sense to me.
Definitely not related to your patches as Michal has the same problem and it's reproducible :-)
I also suspected the problem being me starting with --console parameter, but trying with and without that didn't help isolate the problem either.
Anyway, that patch still doesn't help me get rid of /etc/securetty. The output of 'tty' is still /dev/pts/0 and unless I remove /etc/securetty it doesn't start. What is the output of 'tty' and what ttys do you have in /etc/securetty in your container?
Can you share your xml?
Sure, mine is almost same as yours except the user namespace isolation.
<domain type='lxc'> <name>gentoo</name> <uuid>9de0da50-bddd-40e5-ba4a-24c2ed8fca05</uuid> <memory unit='KiB'>1048576</memory> <currentMemory unit='KiB'>1048576</currentMemory> <vcpu placement='auto'>1</vcpu> <numatune> <memory mode='strict' placement='auto'/> </numatune> <resource> <partition>/machine</partition> </resource> <os> <type arch='x86_64'>exe</type> <init>/sbin/init</init> </os> <features> <acpi/> </features> <clock offset='utc'/> <on_poweroff>destroy</on_poweroff> <on_reboot>restart</on_reboot> <on_crash>destroy</on_crash> <devices> <emulator>/usr/libexec/libvirt_lxc</emulator> <filesystem type='mount' accessmode='passthrough'> <source dir='/mnt/lxc_gentoo'/> <target dir='/'/> </filesystem> <interface type='bridge'> <mac address='52:54:00:ae:8b:47'/> <source bridge='virbr0'/> <target dev='viftestmaster'/> <guest dev='eth0'/> </interface> <console type='pty'> <target type='lxc' port='0'/> </console> </devices> </domain>
This is mine: <domain type='lxc'> <name>gentoo</name> <memory>524288</memory> <os> <type>exe</type> <init>/sbin/init</init> </os> <idmap> <uid start='0' target='100000' count='998'/> <gid start='0' target='100000' count='998'/> <uid start='65533' target='100998' count='2'/> <gid start='65533' target='100998' count='2'/> </idmap> <devices> <console type='pty'/> <filesystem type='mount'> <source dir='/srv/container/gentoo/'/> <target dir='/'/> </filesystem> <interface type='bridge'> <source bridge='br0'/> <mac address='52:54:00:44:55:66'/> </interface> </devices> </domain>
Thanks, //richard
Am 01.07.2015 um 11:40 schrieb Martin Kletzander:
On Tue, Jun 30, 2015 at 07:54:25PM +0200, Richard Weinberger wrote:
Am 30.06.2015 um 19:12 schrieb Martin Kletzander:
Hmm, very strange. What guest container are you using? I tried with a Debian jessi and had user namespace enabled.
Sorry for the late reply. I used simple one. Only gentoo's stage 3 unpacked into a directory, no special settings used for it. Removing /etc/securetty works for me. I'll give it another try, but probably after the freeze. If anyone else wants to review this, don't get stopped by the problems I'm having!
Hmm, just gave gentoo a try, worked perfectly fine.
I tried with latest master with and without your patch. Wtih your patch I got to the problem exactly once even though I tried multiple times. And even though it didn't happen to me at all without your patch, I'm thinking it's just some weird rare race and it's not related to what you've sent. That just wouldn't make sense to me.
I also suspected the problem being me starting with --console parameter, but trying with and without that didn't help isolate the problem either.
--console works fine here.
Anyway, that patch still doesn't help me get rid of /etc/securetty. The output of 'tty' is still /dev/pts/0 and unless I remove /etc/securetty it doesn't start. What is the output of 'tty' and what ttys do you have in /etc/securetty in your container?
tty prints as expected /dev/tty1. (instead of /dev/pts/xy) /etc/securetty is from gentoo, I did not add anything. Thanks, //richard
On Tue, Jun 23, 2015 at 04:38:57PM +0200, Richard Weinberger wrote:
Instead of creating symlinks, bind mount the devices to /dev/pts/XY. Using bind mounts it is no longer needed to add pts devices to files like /dev/securetty.
Signed-off-by: Richard Weinberger <richard@nod.at> --- src/lxc/lxc_container.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-)
I spent ridiculously excessive time on this not working for me, but I just figured out that simple check whether that file is a symlink or not (inside the container) is enough. Ant it works. ACK then, sorry for wasting your time with this as well.
diff --git a/src/lxc/lxc_container.c b/src/lxc/lxc_container.c index 7d531e2..ea76370 100644 --- a/src/lxc/lxc_container.c +++ b/src/lxc/lxc_container.c @@ -1141,6 +1141,20 @@ static int lxcContainerMountFSDevPTS(virDomainDefPtr def, return ret; }
+static int lxcContainerBindMountDevice(const char *src, const char *dst) +{ + if (virFileTouch(dst, 0666) < 0) + return -1; + + if (mount(src, dst, "none", MS_BIND, NULL) < 0) { + virReportSystemError(errno, _("Failed to bind %s on to %s"), src, + dst); + return -1; + } + + return 0; +} + static int lxcContainerSetupDevices(char **ttyPaths, size_t nttyPaths) { size_t i; @@ -1164,34 +1178,24 @@ static int lxcContainerSetupDevices(char **ttyPaths, size_t nttyPaths) }
/* We have private devpts capability, so bind that */ - if (virFileTouch("/dev/ptmx", 0666) < 0) + if (lxcContainerBindMountDevice("/dev/pts/ptmx", "/dev/ptmx") < 0) return -1;
- if (mount("/dev/pts/ptmx", "/dev/ptmx", "ptmx", MS_BIND, NULL) < 0) { - virReportSystemError(errno, "%s", - _("Failed to bind /dev/pts/ptmx on to /dev/ptmx")); - return -1; - } - for (i = 0; i < nttyPaths; i++) { char *tty; if (virAsprintf(&tty, "/dev/tty%zu", i+1) < 0) return -1; - if (symlink(ttyPaths[i], tty) < 0) { - virReportSystemError(errno, - _("Failed to symlink %s to %s"), - ttyPaths[i], tty); - VIR_FREE(tty); + + if (lxcContainerBindMountDevice(ttyPaths[i], tty) < 0) { return -1; + VIR_FREE(tty); } + VIR_FREE(tty); + if (i == 0 && - symlink(ttyPaths[i], "/dev/console") < 0) { - virReportSystemError(errno, - _("Failed to symlink %s to /dev/console"), - ttyPaths[i]); + lxcContainerBindMountDevice(ttyPaths[i], "/dev/console") < 0) return -1; - } } return 0; } -- 2.4.2
-- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
On Fri, Jul 3, 2015 at 1:55 PM, Martin Kletzander <mkletzan@redhat.com> wrote:
On Tue, Jun 23, 2015 at 04:38:57PM +0200, Richard Weinberger wrote:
Instead of creating symlinks, bind mount the devices to /dev/pts/XY. Using bind mounts it is no longer needed to add pts devices to files like /dev/securetty.
Signed-off-by: Richard Weinberger <richard@nod.at> --- src/lxc/lxc_container.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-)
I spent ridiculously excessive time on this not working for me, but I just figured out that simple check whether that file is a symlink or not (inside the container) is enough. Ant it works.
ACK then, sorry for wasting your time with this as well.
diff --git a/src/lxc/lxc_container.c b/src/lxc/lxc_container.c index 7d531e2..ea76370 100644 --- a/src/lxc/lxc_container.c +++ b/src/lxc/lxc_container.c @@ -1141,6 +1141,20 @@ static int lxcContainerMountFSDevPTS(virDomainDefPtr def, return ret; }
+static int lxcContainerBindMountDevice(const char *src, const char *dst) +{ + if (virFileTouch(dst, 0666) < 0) + return -1; + + if (mount(src, dst, "none", MS_BIND, NULL) < 0) { + virReportSystemError(errno, _("Failed to bind %s on to %s"), src, + dst); + return -1; + } + + return 0; +} + static int lxcContainerSetupDevices(char **ttyPaths, size_t nttyPaths) { size_t i; @@ -1164,34 +1178,24 @@ static int lxcContainerSetupDevices(char **ttyPaths, size_t nttyPaths) }
/* We have private devpts capability, so bind that */ - if (virFileTouch("/dev/ptmx", 0666) < 0) + if (lxcContainerBindMountDevice("/dev/pts/ptmx", "/dev/ptmx") < 0) return -1;
- if (mount("/dev/pts/ptmx", "/dev/ptmx", "ptmx", MS_BIND, NULL) < 0) { - virReportSystemError(errno, "%s", - _("Failed to bind /dev/pts/ptmx on to /dev/ptmx")); - return -1; - } - for (i = 0; i < nttyPaths; i++) { char *tty; if (virAsprintf(&tty, "/dev/tty%zu", i+1) < 0) return -1; - if (symlink(ttyPaths[i], tty) < 0) { - virReportSystemError(errno, - _("Failed to symlink %s to %s"), - ttyPaths[i], tty); - VIR_FREE(tty); + + if (lxcContainerBindMountDevice(ttyPaths[i], tty) < 0) { return -1; + VIR_FREE(tty); } + VIR_FREE(tty); + if (i == 0 && - symlink(ttyPaths[i], "/dev/console") < 0) { - virReportSystemError(errno, - _("Failed to symlink %s to /dev/console"), - ttyPaths[i]); + lxcContainerBindMountDevice(ttyPaths[i], "/dev/console") < 0) return -1; - } } return 0; } -- 2.4.2
-- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list
Ping? -- Thanks, //richard
On Tue, Jun 23, 2015 at 04:38:57PM +0200, Richard Weinberger wrote:
Instead of creating symlinks, bind mount the devices to /dev/pts/XY. Using bind mounts it is no longer needed to add pts devices to files like /dev/securetty.
Signed-off-by: Richard Weinberger <richard@nod.at> --- src/lxc/lxc_container.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-)
ACK, no idea why we didn't think of this approach sooner :-) Regards, Daniel -- |: http://berrange.com -o- http://www.flickr.com/photos/dberrange/ :| |: http://libvirt.org -o- http://virt-manager.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: http://entangle-photo.org -o- http://live.gnome.org/gtk-vnc :|
participants (4)
-
Daniel P. Berrange -
Martin Kletzander -
Richard Weinberger -
Richard Weinberger