[libvirt] [PATCH 4/7] Add support for multiple consoles in LXC

From: "Daniel P. Berrange" <berrange(a)redhat.com&gt; Currently the LXC controller only supports setup of a single text console. This is wired up to the container init's stdio, as well as /dev/console and /dev/tty1. Extending support for multiple consoles, means wiring up additional PTYs to /dev/tty2, /dev/tty3, etc, etc. The LXC controller is passed multiple open file handles, one for each console requested. * src/lxc/lxc_container.c, src/lxc/lxc_container.h: Wire up all the /dev/ttyN links required to symlink to /dev/pts/NN * src/lxc/lxc_container.h: Open more container side /dev/pts/NN devices, and adapt event loop to handle I/O from all consoles * src/lxc/lxc_driver.c: Setup multiple host side PTYs --- src/lxc/lxc_container.c | 81 +++++++++++++++--------- src/lxc/lxc_container.h | 3 +- src/lxc/lxc_controller.c | 160 +++++++++++++++++++++++++++++----------------- src/lxc/lxc_driver.c | 62 +++++++++++------- 4 files changed, 191 insertions(+), 115 deletions(-) diff --git a/src/lxc/lxc_container.c b/src/lxc/lxc_container.c index e9891f7..eb0ee6e 100644 --- a/src/lxc/lxc_container.c +++ b/src/lxc/lxc_container.c @@ -90,7 +90,8 @@ struct __lxc_child_argv { unsigned int nveths; char **veths; int monitor; - char *ttyPath; + char **ttyPaths; + size_t nttyPaths; int handshakefd; }; @@ -517,9 +518,9 @@ static int lxcContainerMountDevFS(virDomainFSDefPtr root) return rc; } -static int lxcContainerPopulateDevices(void) +static int lxcContainerPopulateDevices(char **ttyPaths, size_t nttyPaths) { - int i; + size_t i; const struct { int maj; int min; @@ -561,21 +562,28 @@ static int lxcContainerPopulateDevices(void) } } - /* XXX we should allow multiple consoles per container - * for tty2, tty3, etc, but the domain XML does not - * handle this yet - */ - if (symlink("/dev/pts/0", "/dev/tty1") < 0) { - virReportSystemError(errno, "%s", - _("Failed to symlink /dev/pts/0 to /dev/tty1")); - return -1; - } - if (symlink("/dev/pts/0", "/dev/console") < 0) { - virReportSystemError(errno, "%s", - _("Failed to symlink /dev/pts/0 to /dev/console")); - return -1; + for (i = 0 ; i < nttyPaths ; i++) { + char *tty; + if (virAsprintf(&tty, "/dev/tty%zu", i+1) < 0) { + virReportOOMError(); + return -1; + } + if (symlink(ttyPaths[i], tty) < 0) { + VIR_FREE(tty); + virReportSystemError(errno, + _("Failed to symlink %s to %s"), + ttyPaths[i], tty); + return -1; + } + VIR_FREE(tty); + if (i == 0 && + symlink(ttyPaths[i], "/dev/console") < 0) { + virReportSystemError(errno, + _("Failed to symlink %s to /dev/console"), + ttyPaths[i]); + return -1; + } } - return 0; } @@ -904,7 +912,9 @@ static int lxcContainerUnmountOldFS(void) * this is based on this thread http://lkml.org/lkml/2008/3/5/29 */ static int lxcContainerSetupPivotRoot(virDomainDefPtr vmDef, - virDomainFSDefPtr root) + virDomainFSDefPtr root, + char **ttyPaths, + size_t nttyPaths) { /* Gives us a private root, leaving all parent OS mounts on /.oldroot */ if (lxcContainerPivotRoot(root) < 0) @@ -919,7 +929,7 @@ static int lxcContainerSetupPivotRoot(virDomainDefPtr vmDef, return -1; /* Populates device nodes in /dev/ */ - if (lxcContainerPopulateDevices() < 0) + if (lxcContainerPopulateDevices(ttyPaths, nttyPaths) < 0) return -1; /* Sets up any non-root mounts from guest config */ @@ -963,10 +973,12 @@ static int lxcContainerSetupExtraMounts(virDomainDefPtr vmDef) } static int lxcContainerSetupMounts(virDomainDefPtr vmDef, - virDomainFSDefPtr root) + virDomainFSDefPtr root, + char **ttyPaths, + size_t nttyPaths) { if (root) - return lxcContainerSetupPivotRoot(vmDef, root); + return lxcContainerSetupPivotRoot(vmDef, root, ttyPaths, nttyPaths); else return lxcContainerSetupExtraMounts(vmDef); } @@ -1050,17 +1062,25 @@ static int lxcContainerChild( void *data ) root = virDomainGetRootFilesystem(vmDef); - if (root) { - if (virAsprintf(&ttyPath, "%s%s", root->src, argv->ttyPath) < 0) { - virReportOOMError(); - goto cleanup; + if (argv->nttyPaths) { + if (root) { + if (virAsprintf(&ttyPath, "%s%s", root->src, argv->ttyPaths[0]) < 0) { + virReportOOMError(); + goto cleanup; + } + } else { + if (!(ttyPath = strdup(argv->ttyPaths[0]))) { + virReportOOMError(); + goto cleanup; + } } } else { - if (!(ttyPath = strdup(argv->ttyPath))) { + if (!(ttyPath = strdup("/dev/null"))) { virReportOOMError(); goto cleanup; } } + VIR_DEBUG("Container TTY path: %s", ttyPath); ttyfd = open(ttyPath, O_RDWR|O_NOCTTY); @@ -1071,7 +1091,7 @@ static int lxcContainerChild( void *data ) goto cleanup; } - if (lxcContainerSetupMounts(vmDef, root) < 0) + if (lxcContainerSetupMounts(vmDef, root, argv->ttyPaths, argv->nttyPaths) < 0) goto cleanup; if (!virFileExists(vmDef->os.init)) { @@ -1175,14 +1195,15 @@ int lxcContainerStart(virDomainDefPtr def, char **veths, int control, int handshakefd, - char *ttyPath) + char **ttyPaths, + size_t nttyPaths) { pid_t pid; int cflags; int stacksize = getpagesize() * 4; char *stack, *stacktop; - lxc_child_argv_t args = { def, nveths, veths, control, ttyPath, - handshakefd}; + lxc_child_argv_t args = { def, nveths, veths, control, + ttyPaths, nttyPaths, handshakefd}; /* allocate a stack for the container */ if (VIR_ALLOC_N(stack, stacksize) < 0) { diff --git a/src/lxc/lxc_container.h b/src/lxc/lxc_container.h index d6d9b6d..ffeda5e 100644 --- a/src/lxc/lxc_container.h +++ b/src/lxc/lxc_container.h @@ -53,7 +53,8 @@ int lxcContainerStart(virDomainDefPtr def, char **veths, int control, int handshakefd, - char *ttyPath); + char **ttyPaths, + size_t nttyPaths); int lxcContainerAvailable(int features); diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c index fcd57d9..2596387 100644 --- a/src/lxc/lxc_controller.c +++ b/src/lxc/lxc_controller.c @@ -793,20 +793,19 @@ error: */ static int lxcControllerMain(int serverFd, int clientFd, - int hostFd, - int contFd, + int *hostFds, + int *contFds, + size_t nFds, pid_t container) { - struct lxcConsole console = { - .hostFd = hostFd, - .contFd = contFd, - }; + struct lxcConsole *consoles; struct lxcMonitor monitor = { .serverFd = serverFd, .clientFd = clientFd, }; virErrorPtr err; int rc = -1; + size_t i; if (virMutexInit(&lock) < 0) goto cleanup2; @@ -833,8 +832,8 @@ static int lxcControllerMain(int serverFd, goto cleanup; } - VIR_DEBUG("serverFd=%d clientFd=%d hostFd=%d contFd=%d", - serverFd, clientFd, hostFd, contFd); + VIR_DEBUG("serverFd=%d clientFd=%d", + serverFd, clientFd); virResetLastError(); if ((monitor.serverWatch = virEventAddHandle(monitor.serverFd, @@ -858,24 +857,34 @@ static int lxcControllerMain(int serverFd, goto cleanup; } - if ((console.hostWatch = virEventAddHandle(console.hostFd, - VIR_EVENT_HANDLE_READABLE, - lxcConsoleIO, - &console, - NULL)) < 0) { - lxcError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Unable to watch host console PTY")); + if (VIR_ALLOC_N(consoles, nFds) < 0) { + virReportOOMError(); goto cleanup; } - if ((console.contWatch = virEventAddHandle(console.contFd, - VIR_EVENT_HANDLE_READABLE, - lxcConsoleIO, - &console, - NULL)) < 0) { - lxcError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Unable to watch host console PTY")); - goto cleanup; + for (i = 0 ; i < nFds ; i++) { + consoles[i].hostFd = hostFds[i]; + consoles[i].contFd = contFds[i]; + + if ((consoles[i].hostWatch = virEventAddHandle(consoles[i].hostFd, + VIR_EVENT_HANDLE_READABLE, + lxcConsoleIO, + &consoles[i], + NULL)) < 0) { + lxcError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to watch host console PTY")); + goto cleanup; + } + + if ((consoles[i].contWatch = virEventAddHandle(consoles[i].contFd, + VIR_EVENT_HANDLE_READABLE, + lxcConsoleIO, + &consoles[i], + NULL)) < 0) { + lxcError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to watch host console PTY")); + goto cleanup; + } } while (!quit) { @@ -891,10 +900,9 @@ cleanup: virMutexDestroy(&lock); signal(SIGCHLD, SIG_DFL); cleanup2: - VIR_FORCE_CLOSE(console.hostFd); - VIR_FORCE_CLOSE(console.contFd); VIR_FORCE_CLOSE(monitor.serverFd); VIR_FORCE_CLOSE(monitor.clientFd); + VIR_FREE(consoles); return rc; } @@ -1019,14 +1027,15 @@ lxcControllerRun(virDomainDefPtr def, char **veths, int monitor, int client, - int appPty, + int *ttyFDs, + size_t nttyFDs, int handshakefd) { int rc = -1; int control[2] = { -1, -1}; int containerhandshake[2] = { -1, -1 }; - int containerPty = -1; - char *containerPtyPath = NULL; + int *containerTtyFDs = NULL; + char **containerTtyPaths = NULL; pid_t container = -1; virDomainFSDefPtr root; char *devpts = NULL; @@ -1035,6 +1044,15 @@ lxcControllerRun(virDomainDefPtr def, int *loopDevs = NULL; size_t i; + if (VIR_ALLOC_N(containerTtyFDs, nttyFDs) < 0) { + virReportOOMError(); + goto cleanup; + } + if (VIR_ALLOC_N(containerTtyPaths, nttyFDs) < 0) { + virReportOOMError(); + goto cleanup; + } + if (socketpair(PF_UNIX, SOCK_STREAM, 0, control) < 0) { virReportSystemError(errno, "%s", _("sockpair failed")); @@ -1125,26 +1143,36 @@ lxcControllerRun(virDomainDefPtr def, VIR_WARN("Kernel does not support private devpts, using shared devpts"); VIR_FREE(devptmx); } - } - - if (devptmx) { - VIR_DEBUG("Opening tty on private %s", devptmx); - if (lxcCreateTty(devptmx, &containerPty, &containerPtyPath) < 0) { - virReportSystemError(errno, "%s", - _("Failed to allocate tty")); - goto cleanup; - } } else { - VIR_DEBUG("Opening tty on shared /dev/ptmx"); - if (virFileOpenTty(&containerPty, - &containerPtyPath, - 0) < 0) { - virReportSystemError(errno, "%s", - _("Failed to allocate tty")); + if (nttyFDs != -1) { + lxcError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("Expected exactly one TTY fd")); goto cleanup; } } + for (i = 0 ; i < nttyFDs ; i++) { + if (devptmx) { + VIR_DEBUG("Opening tty on private %s", devptmx); + if (lxcCreateTty(devptmx, + &containerTtyFDs[i], + &containerTtyPaths[i]) < 0) { + virReportSystemError(errno, "%s", + _("Failed to allocate tty")); + goto cleanup; + } + } else { + VIR_DEBUG("Opening tty on shared /dev/ptmx"); + if (virFileOpenTty(&containerTtyFDs[i], + &containerTtyPaths[i], + 0) < 0) { + virReportSystemError(errno, "%s", + _("Failed to allocate tty")); + goto cleanup; + } + } + } + if (lxcSetPersonality(def) < 0) goto cleanup; @@ -1153,7 +1181,8 @@ lxcControllerRun(virDomainDefPtr def, veths, control[1], containerhandshake[1], - containerPtyPath)) < 0) + containerTtyPaths, + nttyFDs)) < 0) goto cleanup; VIR_FORCE_CLOSE(control[1]); VIR_FORCE_CLOSE(containerhandshake[1]); @@ -1192,28 +1221,39 @@ lxcControllerRun(virDomainDefPtr def, VIR_FORCE_CLOSE(handshakefd); if (virSetBlocking(monitor, false) < 0 || - virSetBlocking(client, false) < 0 || - virSetBlocking(appPty, false) < 0 || - virSetBlocking(containerPty, false) < 0) { + virSetBlocking(client, false) < 0) { virReportSystemError(errno, "%s", _("Unable to set file descriptor non blocking")); goto cleanup; } + for (i = 0 ; i < nttyFDs ; i++) { + if (virSetBlocking(ttyFDs[i], false) < 0 || + virSetBlocking(containerTtyFDs[i], false) < 0) { + virReportSystemError(errno, "%s", + _("Unable to set file descriptor non blocking")); + goto cleanup; + } + } - rc = lxcControllerMain(monitor, client, appPty, containerPty, container); - monitor = client = appPty = containerPty = -1; + rc = lxcControllerMain(monitor, client, ttyFDs, containerTtyFDs, nttyFDs, container); + monitor = client = -1; cleanup: VIR_FREE(devptmx); VIR_FREE(devpts); VIR_FORCE_CLOSE(control[0]); VIR_FORCE_CLOSE(control[1]); - VIR_FREE(containerPtyPath); - VIR_FORCE_CLOSE(containerPty); VIR_FORCE_CLOSE(handshakefd); VIR_FORCE_CLOSE(containerhandshake[0]); VIR_FORCE_CLOSE(containerhandshake[1]); + for (i = 0 ; i < nttyFDs ; i++) + VIR_FREE(containerTtyPaths[i]); + VIR_FREE(containerTtyPaths); + for (i = 0 ; i < nttyFDs ; i++) + VIR_FORCE_CLOSE(containerTtyFDs[i]); + VIR_FREE(containerTtyFDs); + for (i = 0 ; i < nloopDevs ; i++) VIR_FORCE_CLOSE(loopDevs[i]); VIR_FREE(loopDevs); @@ -1239,7 +1279,6 @@ int main(int argc, char *argv[]) int nveths = 0; char **veths = NULL; int monitor = -1; - int appPty = -1; int handshakefd = -1; int bg = 0; virCapsPtr caps = NULL; @@ -1255,6 +1294,8 @@ int main(int argc, char *argv[]) { "help", 0, NULL, 'h' }, { 0, 0, 0, 0 }, }; + int *ttyFDs = NULL; + size_t nttyFDs = 0; if (setlocale(LC_ALL, "") == NULL || bindtextdomain(PACKAGE, LOCALEDIR) == NULL || @@ -1296,7 +1337,11 @@ int main(int argc, char *argv[]) break; case 'c': - if (virStrToLong_i(optarg, NULL, 10, &appPty) < 0) { + if (VIR_REALLOC_N(ttyFDs, nttyFDs + 1) < 0) { + virReportOOMError(); + goto cleanup; + } + if (virStrToLong_i(optarg, NULL, 10, &ttyFDs[nttyFDs++]) < 0) { fprintf(stderr, "malformed --console argument '%s'", optarg); goto cleanup; } @@ -1334,11 +1379,6 @@ int main(int argc, char *argv[]) goto cleanup; } - if (appPty < 0) { - fprintf(stderr, "%s: missing --console argument for container PTY\n", argv[0]); - goto cleanup; - } - if (handshakefd < 0) { fprintf(stderr, "%s: missing --handshake argument for container PTY\n", argv[0]); @@ -1418,8 +1458,8 @@ int main(int argc, char *argv[]) goto cleanup; } - rc = lxcControllerRun(def, nveths, veths, monitor, client, appPty, - handshakefd); + rc = lxcControllerRun(def, nveths, veths, monitor, client, + ttyFDs, nttyFDs, handshakefd); cleanup: diff --git a/src/lxc/lxc_driver.c b/src/lxc/lxc_driver.c index 9b5c9db..8e02676 100644 --- a/src/lxc/lxc_driver.c +++ b/src/lxc/lxc_driver.c @@ -1451,11 +1451,12 @@ lxcBuildControllerCmd(lxc_driver_t *driver, virDomainObjPtr vm, int nveths, char **veths, - int appPty, + int *ttyFDs, + size_t nttyFDs, int logfile, int handshakefd) { - int i; + size_t i; char *filterstr; char *outputstr; virCommandPtr cmd; @@ -1496,8 +1497,12 @@ lxcBuildControllerCmd(lxc_driver_t *driver, virLogGetDefaultPriority()); } - virCommandAddArgList(cmd, "--name", vm->def->name, "--console", NULL); - virCommandAddArgFormat(cmd, "%d", appPty); + virCommandAddArgList(cmd, "--name", vm->def->name, NULL); + for (i = 0 ; i < nttyFDs ; i++) { + virCommandAddArg(cmd, "--console"); + virCommandAddArgFormat(cmd, "%d", ttyFDs[i]); + virCommandPreserveFD(cmd, ttyFDs[i]); + } virCommandAddArg(cmd, "--handshake"); virCommandAddArgFormat(cmd, "%d", handshakefd); virCommandAddArg(cmd, "--background"); @@ -1522,7 +1527,6 @@ lxcBuildControllerCmd(lxc_driver_t *driver, goto cleanup; } - virCommandPreserveFD(cmd, appPty); virCommandPreserveFD(cmd, handshakefd); virCommandSetOutputFD(cmd, &logfile); virCommandSetErrorFD(cmd, &logfile); @@ -1625,9 +1629,9 @@ static int lxcVmStart(virConnectPtr conn, virDomainRunningReason reason) { int rc = -1, r; - unsigned int i; - int parentTty; - char *parentTtyPath = NULL; + size_t nttyFDs = 0; + int *ttyFDs = NULL; + size_t i; char *logfile = NULL; int logfd = -1; unsigned int nveths = 0; @@ -1677,26 +1681,34 @@ static int lxcVmStart(virConnectPtr conn, return -1; } - /* open parent tty */ - if (virFileOpenTty(&parentTty, &parentTtyPath, 1) < 0) { - virReportSystemError(errno, "%s", - _("Failed to allocate tty")); + /* Here we open all the PTYs we need on the host OS side. + * The LXC controller will open the guest OS side PTYs + * forward I/O between them. + */ + nttyFDs = vm->def->nconsoles; + if (VIR_ALLOC_N(ttyFDs, nttyFDs) < 0) { + virReportOOMError(); goto cleanup; } - if (vm->def->nconsoles) { - if (vm->def->nconsoles > 1) { + for (i = 0 ; i < vm->def->nconsoles ; i++) + ttyFDs[i] = -1; + + for (i = 0 ; i < vm->def->nconsoles ; i++) { + char *ttyPath; + if (vm->def->consoles[i]->source.type != VIR_DOMAIN_CHR_TYPE_PTY) { lxcError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("Only one console supported")); + _("Only PTY console types are supported")); goto cleanup; } - if (vm->def->consoles[0]->source.type == VIR_DOMAIN_CHR_TYPE_PTY) { - VIR_FREE(vm->def->consoles[0]->source.data.file.path); - vm->def->consoles[0]->source.data.file.path = parentTtyPath; - } else { - VIR_FREE(parentTtyPath); + + if (virFileOpenTty(&ttyFDs[i], &ttyPath, 1) < 0) { + virReportSystemError(errno, "%s", + _("Failed to allocate tty")); + goto cleanup; } - } else { - VIR_FREE(parentTtyPath); + + VIR_FREE(vm->def->consoles[i]->source.data.file.path); + vm->def->consoles[i]->source.data.file.path = ttyPath; } if (lxcSetupInterfaces(conn, vm->def, &nveths, &veths) != 0) @@ -1723,7 +1735,8 @@ static int lxcVmStart(virConnectPtr conn, if (!(cmd = lxcBuildControllerCmd(driver, vm, nveths, veths, - parentTty, logfd, handshakefds[1]))) + ttyFDs, nttyFDs, + logfd, handshakefds[1]))) goto cleanup; /* Log timestamp */ @@ -1825,7 +1838,8 @@ cleanup: VIR_FORCE_CLOSE(priv->monitor); virDomainConfVMNWFilterTeardown(vm); } - VIR_FORCE_CLOSE(parentTty); + for (i = 0 ; i < nttyFDs ; i++) + VIR_FORCE_CLOSE(ttyFDs[i]); VIR_FORCE_CLOSE(handshakefds[0]); VIR_FORCE_CLOSE(handshakefds[1]); VIR_FREE(logfile); -- 1.7.6.4