This patch does some simple re-factoring of the way the TTYs and control socket are handled to reduce the amount of state stored in the lxc_vm_t structure, in preparation for the switchover to the generic domain handling APIs. lxc_conf.c | 1 lxc_conf.h | 12 --- lxc_container.c | 60 +++++++-------- lxc_container.h | 14 +++ lxc_driver.c | 213 ++++++++++++++++---------------------------------------- 5 files changed, 104 insertions(+), 196 deletions(-) Daniel diff -r 63b8398c302e src/lxc_conf.c --- a/src/lxc_conf.c Mon Jul 14 12:18:23 2008 +0100 +++ b/src/lxc_conf.c Tue Jul 15 11:55:48 2008 +0100 @@ -1041,7 +1041,6 @@ void lxcFreeVM(lxc_vm_t *vm) { lxcFreeVMDef(vm->def); - VIR_FREE(vm->containerTty); VIR_FREE(vm); } diff -r 63b8398c302e src/lxc_conf.h --- a/src/lxc_conf.h Mon Jul 14 12:18:23 2008 +0100 +++ b/src/lxc_conf.h Tue Jul 15 11:55:48 2008 +0100 @@ -35,12 +35,6 @@ #define LXC_MAX_XML_LENGTH 16384 #define LXC_MAX_ERROR_LEN 1024 #define LXC_DOMAIN_TYPE "lxc" -#define LXC_PARENT_SOCKET 0 -#define LXC_CONTAINER_SOCKET 1 - -/* messages between parent and container */ -typedef char lxc_message_t; -#define LXC_CONTINUE_MSG 'c' /* types of networks for containers */ enum lxc_net_type { @@ -98,12 +92,6 @@ char configFileBase[PATH_MAX]; char ttyPidFile[PATH_MAX]; - - int parentTty; - int containerTtyFd; - char *containerTty; - - int sockpair[2]; lxc_vm_def_t *def; diff -r 63b8398c302e src/lxc_container.c --- a/src/lxc_container.c Mon Jul 14 12:18:23 2008 +0100 +++ b/src/lxc_container.c Tue Jul 15 11:55:48 2008 +0100 @@ -33,7 +33,6 @@ #include <unistd.h> #include "lxc_container.h" -#include "lxc_conf.h" #include "util.h" #include "memory.h" #include "veth.h" @@ -83,10 +82,11 @@ * * Returns 0 on success or -1 in case of error */ -static int lxcSetContainerStdio(const char *ttyName) +static int lxcSetContainerStdio(const char *ttyPath) { int rc = -1; int ttyfd; + int open_max, i; if (setsid() < 0) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, @@ -94,10 +94,10 @@ goto error_out; } - ttyfd = open(ttyName, O_RDWR|O_NOCTTY); + ttyfd = open(ttyPath, O_RDWR|O_NOCTTY); if (ttyfd < 0) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("open(%s) failed: %s"), ttyName, strerror(errno)); + _("open(%s) failed: %s"), ttyPath, strerror(errno)); goto error_out; } @@ -107,7 +107,12 @@ goto cleanup; } - close(0); close(1); close(2); + /* Just in case someone forget to set FD_CLOEXEC, explicitly + * close all FDs before executing the container */ + open_max = sysconf (_SC_OPEN_MAX); + for (i = 0; i < open_max; i++) + if (i != ttyfd) + close(i); if (dup2(ttyfd, 0) < 0) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, @@ -144,12 +149,11 @@ * * Returns 0 on success or -1 in case of error */ -static int lxcExecWithTty(lxc_vm_t *vm) +static int lxcExecWithTty(lxc_vm_def_t *vmDef, char *ttyPath) { int rc = -1; - lxc_vm_def_t *vmDef = vm->def; - if(lxcSetContainerStdio(vm->containerTty) < 0) { + if(lxcSetContainerStdio(ttyPath) < 0) { goto exit_with_error; } @@ -161,7 +165,7 @@ /** * lxcWaitForContinue: - * @vm: Pointer to vm structure + * @monitor: monitor FD from parent * * This function will wait for the container continue message from the * parent process. It will send this message on the socket pair stored in @@ -169,31 +173,23 @@ * * Returns 0 on success or -1 in case of error */ -static int lxcWaitForContinue(lxc_vm_t *vm) +static int lxcWaitForContinue(int monitor) { - int rc = -1; lxc_message_t msg; int readLen; - readLen = saferead(vm->sockpair[LXC_CONTAINER_SOCKET], &msg, sizeof(msg)); - if (readLen != sizeof(msg)) { + readLen = saferead(monitor, &msg, sizeof(msg)); + if (readLen != sizeof(msg) || + msg != LXC_CONTINUE_MSG) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, _("Failed to read the container continue message: %s"), strerror(errno)); - goto error_out; + return -1; } DEBUG0("Received container continue message"); - close(vm->sockpair[LXC_PARENT_SOCKET]); - vm->sockpair[LXC_PARENT_SOCKET] = -1; - close(vm->sockpair[LXC_CONTAINER_SOCKET]); - vm->sockpair[LXC_CONTAINER_SOCKET] = -1; - - rc = 0; - -error_out: - return rc; + return 0; } /** @@ -204,12 +200,12 @@ * * Returns 0 on success or nonzero in case of error */ -static int lxcEnableInterfaces(const lxc_vm_t *vm) +static int lxcEnableInterfaces(const lxc_vm_def_t *def) { int rc = 0; const lxc_net_def_t *net; - for (net = vm->def->nets; net; net = net->next) { + for (net = def->nets; net; net = net->next) { DEBUG("Enabling %s", net->containerVeth); rc = vethInterfaceUpOrDown(net->containerVeth, 1); if (0 != rc) { @@ -218,7 +214,7 @@ } /* enable lo device only if there were other net devices */ - if (vm->def->nets) + if (def->nets) rc = vethInterfaceUpOrDown("lo", 1); error_out: @@ -237,11 +233,11 @@ * * Returns 0 on success or -1 in case of error */ -int lxcChild( void *argv ) +int lxcChild( void *data ) { int rc = -1; - lxc_vm_t *vm = (lxc_vm_t *)argv; - lxc_vm_def_t *vmDef = vm->def; + lxc_child_argv_t *argv = data; + lxc_vm_def_t *vmDef = argv->config; lxc_mount_t *curMount; int i; @@ -278,16 +274,16 @@ } /* Wait for interface devices to show up */ - if (0 != (rc = lxcWaitForContinue(vm))) { + if (0 != (rc = lxcWaitForContinue(argv->monitor))) { goto cleanup; } /* enable interfaces */ - if (0 != (rc = lxcEnableInterfaces(vm))) { + if (0 != (rc = lxcEnableInterfaces(vmDef))) { goto cleanup; } - rc = lxcExecWithTty(vm); + rc = lxcExecWithTty(vmDef, argv->ttyPath); /* this function will only return if an error occured */ cleanup: diff -r 63b8398c302e src/lxc_container.h --- a/src/lxc_container.h Mon Jul 14 12:18:23 2008 +0100 +++ b/src/lxc_container.h Tue Jul 15 11:55:48 2008 +0100 @@ -24,7 +24,21 @@ #ifndef LXC_CONTAINER_H #define LXC_CONTAINER_H +#include "lxc_conf.h" + #ifdef WITH_LXC + +typedef struct __lxc_child_argv lxc_child_argv_t; +struct __lxc_child_argv { + lxc_vm_def_t *config; + int monitor; + char *ttyPath; +}; + +/* messages between parent and container */ +typedef char lxc_message_t; +#define LXC_CONTINUE_MSG 'c' + /* Function declarations */ int lxcChild( void *argv ); diff -r 63b8398c302e src/lxc_driver.c --- a/src/lxc_driver.c Mon Jul 14 12:18:23 2008 +0100 +++ b/src/lxc_driver.c Tue Jul 15 11:55:48 2008 +0100 @@ -561,27 +561,23 @@ /** * lxcSendContainerContinue: - * @vm: pointer to virtual machine structure + * @monitor: FD for communicating with child * * Sends the continue message via the socket pair stored in the vm * structure. * * Returns 0 on success or -1 in case of error */ -static int lxcSendContainerContinue(const lxc_vm_t *vm) +static int lxcSendContainerContinue(virConnectPtr conn, + int monitor) { int rc = -1; lxc_message_t msg = LXC_CONTINUE_MSG; int writeCount = 0; - if (NULL == vm) { - goto error_out; - } - - writeCount = safewrite(vm->sockpair[LXC_PARENT_SOCKET], &msg, - sizeof(msg)); + writeCount = safewrite(monitor, &msg, sizeof(msg)); if (writeCount != sizeof(msg)) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, _("unable to send container continue message: %s"), strerror(errno)); goto error_out; @@ -605,12 +601,15 @@ */ static int lxcStartContainer(virConnectPtr conn, lxc_driver_t* driver, - lxc_vm_t *vm) + lxc_vm_t *vm, + int monitor, + char *ttyPath) { int rc = -1; int flags; int stacksize = getpagesize() * 4; char *stack, *stacktop; + lxc_child_argv_t args = { vm->def, monitor, ttyPath }; /* allocate a stack for the container */ if (VIR_ALLOC_N(stack, stacksize) < 0) { @@ -625,7 +624,7 @@ if (vm->def->nets != NULL) flags |= CLONE_NEWNET; - vm->def->id = clone(lxcChild, stacktop, flags, (void *)vm); + vm->def->id = clone(lxcChild, stacktop, flags, &args); DEBUG("clone() returned, %d", vm->def->id); @@ -643,117 +642,9 @@ return rc; } -/** - * lxcPutTtyInRawMode: - * @conn: pointer to connection - * @ttyDev: file descriptor for tty - * - * Sets tty attributes via cfmakeraw() - * - * Returns 0 on success or -1 in case of error - */ -static int lxcPutTtyInRawMode(virConnectPtr conn, int ttyDev) -{ - int rc = -1; - - struct termios ttyAttr; - - if (tcgetattr(ttyDev, &ttyAttr) < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - "tcgetattr() failed: %s", strerror(errno)); - goto cleanup; - } - - cfmakeraw(&ttyAttr); - - if (tcsetattr(ttyDev, TCSADRAIN, &ttyAttr) < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - "tcsetattr failed: %s", strerror(errno)); - goto cleanup; - } - - rc = 0; - -cleanup: - return rc; -} /** - * lxcSetupTtyTunnel: - * @conn: pointer to connection - * @vmDef: pointer to virtual machine definition structure - * @ttyDev: pointer to int. On success will be set to fd for master - * end of tty - * - * Opens and configures the parent side tty - * - * Returns 0 on success or -1 in case of error - */ -static int lxcSetupTtyTunnel(virConnectPtr conn, - lxc_vm_def_t *vmDef, - int* ttyDev) -{ - int rc = -1; - char *ptsStr; - - if (0 < strlen(vmDef->tty)) { - *ttyDev = posix_openpt(O_RDWR|O_NOCTTY|O_NONBLOCK); - if (*ttyDev < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - "open() tty failed: %s", strerror(errno)); - goto setup_complete; - } - - rc = grantpt(*ttyDev); - if (rc < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - "grantpt() failed: %s", strerror(errno)); - goto setup_complete; - } - - rc = unlockpt(*ttyDev); - if (rc < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - "unlockpt() failed: %s", strerror(errno)); - goto setup_complete; - } - - /* get the name and print it to stdout */ - ptsStr = ptsname(*ttyDev); - if (ptsStr == NULL) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - "ptsname() failed"); - goto setup_complete; - } - /* This value needs to be stored in the container configuration file */ - VIR_FREE(vmDef->tty); - if (!(vmDef->tty = strdup(ptsStr))) { - lxcError(conn, NULL, VIR_ERR_NO_MEMORY, - _("unable to get storage for vm tty name")); - goto setup_complete; - } - - /* Enter raw mode, so all characters are passed directly to child */ - if (lxcPutTtyInRawMode(conn, *ttyDev) < 0) { - goto setup_complete; - } - - } else { - *ttyDev = -1; - } - - rc = 0; - -setup_complete: - if((0 != rc) && (*ttyDev > 0)) { - close(*ttyDev); - } - - return rc; -} - -/** - * lxcSetupContainerTty: + * lxcOpenTty: * @conn: pointer to connection * @ttymaster: pointer to int. On success, set to fd for master end * @ttyName: On success, will point to string slave end of tty. Caller @@ -763,12 +654,12 @@ * * Returns 0 on success or -1 in case of error */ -static int lxcSetupContainerTty(virConnectPtr conn, - int *ttymaster, - char **ttyName) +static int lxcOpenTty(virConnectPtr conn, + int *ttymaster, + char **ttyName, + int rawmode) { int rc = -1; - char tempTtyName[PATH_MAX]; *ttymaster = posix_openpt(O_RDWR|O_NOCTTY|O_NONBLOCK); if (*ttymaster < 0) { @@ -783,27 +674,43 @@ goto cleanup; } - if (0 != ptsname_r(*ttymaster, tempTtyName, sizeof(tempTtyName))) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - _("ptsname_r failed: %s"), strerror(errno)); - goto cleanup; + if (rawmode) { + struct termios ttyAttr; + if (tcgetattr(*ttymaster, &ttyAttr) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + "tcgetattr() failed: %s", strerror(errno)); + goto cleanup; + } + + cfmakeraw(&ttyAttr); + + if (tcsetattr(*ttymaster, TCSADRAIN, &ttyAttr) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + "tcsetattr failed: %s", strerror(errno)); + goto cleanup; + } } - if (VIR_ALLOC_N(*ttyName, strlen(tempTtyName) + 1) < 0) { - lxcError(conn, NULL, VIR_ERR_NO_MEMORY, - _("unable to allocate container name string")); - goto cleanup; + if (ttyName) { + char tempTtyName[PATH_MAX]; + if (0 != ptsname_r(*ttymaster, tempTtyName, sizeof(tempTtyName))) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("ptsname_r failed: %s"), strerror(errno)); + goto cleanup; + } + + if ((*ttyName = strdup(tempTtyName)) == NULL) { + lxcError(conn, NULL, VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } } - - strcpy(*ttyName, tempTtyName); rc = 0; cleanup: - if (0 != rc) { - if (-1 != *ttymaster) { - close(*ttymaster); - } + if (rc != 0 && + *ttymaster != -1) { + close(*ttymaster); } return rc; @@ -989,15 +896,18 @@ lxc_vm_t * vm) { int rc = -1; - lxc_vm_def_t *vmDef = vm->def; + int sockpair[2]; + int containerTty, parentTty; + char *containerTtyPath = NULL; /* open parent tty */ - if (lxcSetupTtyTunnel(conn, vmDef, &vm->parentTty) < 0) { + VIR_FREE(vm->def->tty); + if (lxcOpenTty(conn, &parentTty, &vm->def->tty, 1) < 0) { goto cleanup; } /* open container tty */ - if (lxcSetupContainerTty(conn, &(vm->containerTtyFd), &(vm->containerTty)) < 0) { + if (lxcOpenTty(conn, &containerTty, &containerTtyPath, 0) < 0) { goto cleanup; } @@ -1011,15 +921,15 @@ if (vm->pid == 0) { /* child process calls forward routine */ - lxcTtyForward(vm->parentTty, vm->containerTtyFd); + lxcTtyForward(parentTty, containerTty); } if (lxcStoreTtyPid(driver, vm)) { DEBUG0("unable to store tty pid"); } - close(vm->parentTty); - close(vm->containerTtyFd); + close(parentTty); + close(containerTty); if (0 != (rc = lxcSetupInterfaces(conn, vm))) { goto cleanup; @@ -1027,7 +937,7 @@ /* create a socket pair to send continue message to the container once */ /* we've completed the post clone configuration */ - if (0 != socketpair(PF_UNIX, SOCK_STREAM, 0, vm->sockpair)) { + if (0 != socketpair(PF_UNIX, SOCK_STREAM, 0, sockpair)) { lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, _("sockpair failed: %s"), strerror(errno)); goto cleanup; @@ -1035,7 +945,9 @@ /* check this rc */ - rc = lxcStartContainer(conn, driver, vm); + rc = lxcStartContainer(conn, driver, vm, + sockpair[1], + containerTtyPath); if (rc != 0) goto cleanup; @@ -1043,7 +955,7 @@ if (rc != 0) goto cleanup; - rc = lxcSendContainerContinue(vm); + rc = lxcSendContainerContinue(conn, sockpair[0]); if (rc != 0) goto cleanup; @@ -1052,10 +964,9 @@ driver->nactivevms++; cleanup: - close(vm->sockpair[LXC_PARENT_SOCKET]); - vm->sockpair[LXC_PARENT_SOCKET] = -1; - close(vm->sockpair[LXC_CONTAINER_SOCKET]); - vm->sockpair[LXC_CONTAINER_SOCKET] = -1; + close(sockpair[0]); + close(sockpair[1]); + VIR_FREE(containerTtyPath); return rc; } -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|

Every LXC container has 2 processes, the leader of the actual container and a helper process to do the I/O forwarding. The LXC driver is written to allow libvirtd to be restarted without needing to shutdown the active containers. For this it uses a PID file in /var/run/libvirt/lxc/NAME.pid The driver also uses SIGCHLD to detect when a container has terminated. Here-in lies the problem - if you restart libvirtd, the container process gets re-parented to init, and so libvirtd will never get any further SIGCHLD signals for it. This patch attempts to address this problem by changing the relationship between libvirtd and the container processes. Instad of there being two processes which are siblings, the I/O process becomes the parent of the actual container. So the general idea is: - libvirtd LXC driver spawns a 'controller' process - this immediately double-forks itself into the background, making itself process leader, changing its root directory to /, and redirecting its stdin to /dev/null and its stdout/err to /var/log/libvirt/lxc/NAME.log. - The 'controller' process inherits a UNIX domain socket from libvirtd which has had bind() called against /var/run/libvirt/lxc/NAME.sock - Once it has backgrounded itself, the controller calls accept() on the socket, blocking until a client connects. If this fails for any reason the controller exits. - THe 'controller' also writes a file to /var/run/libvirt/lxc/NAME.pid containing its PID. - Immediately after forking the 'controller' process, the libvirtd LXC driver, calls connect() on /var/run/libvirt/lxc/NAME.sock. And does a blocking read of sizeof(pid_t) bytes. - They now do a handshake, consisting of simply sending & receving a single byte. This basically is to ensure the libvirt driver blocks until the controller has finished writing its PID file At this point in time, the libvirtd LXC driver knows what the controller process' PID is. This becomes the 'ID' of the virDomain object associated with this. If anything goes wrong from here-on, the libvirtd LXC driver also now knows what PID it has to kill off. The UNIX socket to the controller process is kept open, and registered with the libvirtd event loop for POLLHUP events. This means the LXC driver can get notification when the controller terminates, without needing to rely on SIGCHLD events. - Now the 'controller' has told libvirtd what its PID is, it goes ahead and starts the real 'container' process. - When the 'container' is up and running, the controller goes into the event loop where it handles I/O from the PTYs. It also keeps an eye out for SIGHUP on the client socket to the libvirtd daemon. - If the client goes away, it'll know it needs to accept() a new client (ie the libvirtd daemon starting up again). Upon a new client connecting it'll do the PID handshake again, so that libvirtd knows what the container ID is again. - When libvirtd starts up, it reads the container configs from /etc/libvirt/lxc and for each one, tries to connect to /var/run/libvirt/lxc/NAME.sock, and read the PID file. This lets it figure out which containers are running. Notice that throughout this, libvirtd's LXC driver doesn't need to know the PID of the actual container process - only that of the controller process. Think of the controller as serving the equivalent role of QEMU in context of KVM. QEMU provides the backend device model for KVM. When QEMU dies, the guest domain goes away. Well the controller provides the 'backend' device model for the container. - though in this case it really only the text console backend. When the controller dies, the container goes away. So architecturally the LXC driver is now very closely aligned with the QEMU driver. The main differences are that LXC can handle libvirtd restarts, and that the LXC driver simply forks() the controller. One could imagine these distinctions going away - the QEMU driver can get a restart capability in much the same way as the LXC restart works. We may want to make the controller process a proper standalone binary which can be directly exec'd, rather than just forked off from libvirtd. lxc_conf.c | 195 ---------------- lxc_conf.h | 12 lxc_container.c | 39 +-- lxc_container.h | 8 lxc_controller.c | 349 +++++++++++++++++++++++++++- lxc_controller.h | 12 lxc_driver.c | 662 ++++++++++++++++++++++++++----------------------------- util.c | 158 +++++++++++++ util.h | 13 + 9 files changed, 870 insertions(+), 578 deletions(-) Daniel diff -r 8093fb566748 src/lxc_conf.c --- a/src/lxc_conf.c Fri Aug 01 14:47:33 2008 +0100 +++ b/src/lxc_conf.c Tue Aug 05 12:13:24 2008 +0100 @@ -833,25 +833,24 @@ strncpy(vm->configFileBase, file, PATH_MAX); vm->configFile[PATH_MAX-1] = '\0'; - if (lxcLoadTtyPid(driver, vm) < 0) { - DEBUG0("failed to load tty pid"); - } - return vm; } int lxcLoadDriverConfig(lxc_driver_t *driver) { /* Set the container configuration directory */ - driver->configDir = strdup(SYSCONF_DIR "/libvirt/lxc"); - if (NULL == driver->configDir) { - lxcError(NULL, NULL, VIR_ERR_NO_MEMORY, "configDir"); - return -1; - } - - driver->stateDir = strdup(LOCAL_STATE_DIR "/run/libvirt/lxc"); + if ((driver->configDir = strdup(SYSCONF_DIR "/libvirt/lxc")) == NULL) + goto no_memory; + if ((driver->stateDir = strdup(LOCAL_STATE_DIR "/run/libvirt/lxc")) == NULL) + goto no_memory; + if ((driver->logDir = strdup(LOCAL_STATE_DIR "/log/libvirt/lxc")) == NULL) + goto no_memory; return 0; + +no_memory: + lxcError(NULL, NULL, VIR_ERR_NO_MEMORY, "configDir"); + return -1; } int lxcLoadContainerConfigFile(lxc_driver_t *driver, @@ -1012,9 +1011,7 @@ curNet = vmdef->nets; while (curNet) { nextNet = curNet->next; - printf("Freeing %s:%s\n", curNet->parentVeth, curNet->containerVeth); VIR_FREE(curNet->parentVeth); - VIR_FREE(curNet->containerVeth); VIR_FREE(curNet->txName); VIR_FREE(curNet); curNet = nextNet; @@ -1106,176 +1103,4 @@ return 0; } -/** - * lxcStoreTtyPid: - * @driver: pointer to driver - * @vm: Ptr to VM - * - * Stores the pid of the tty forward process contained in vm->pid - * LOCAL_STATE_DIR/run/libvirt/lxc/{container_name}.pid - * - * Returns 0 on success or -1 in case of error - */ -int lxcStoreTtyPid(const lxc_driver_t *driver, lxc_vm_t *vm) -{ - int rc = -1; - int fd; - FILE *file = NULL; - - if (vm->ttyPidFile[0] == 0x00) { - if ((rc = virFileMakePath(driver->stateDir))) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot create lxc state directory %s: %s"), - driver->stateDir, strerror(rc)); - goto error_out; - } - - if (virFileBuildPath(driver->stateDir, vm->def->name, ".pid", - vm->ttyPidFile, PATH_MAX) < 0) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot construct tty pid file path")); - goto error_out; - } - } - - if ((fd = open(vm->ttyPidFile, - O_WRONLY | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR)) < 0) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot create tty pid file %s: %s"), - vm->ttyPidFile, strerror(errno)); - goto error_out; - } - - if (!(file = fdopen(fd, "w"))) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot fdopen tty pid file %s: %s"), - vm->ttyPidFile, strerror(errno)); - - if (close(fd) < 0) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("failed to close tty pid file %s: %s"), - vm->ttyPidFile, strerror(errno)); - } - - goto error_out; - } - - if (fprintf(file, "%d", vm->pid) < 0) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot write tty pid file %s: %s"), - vm->ttyPidFile, strerror(errno)); - - goto fclose_error_out; - } - - rc = 0; - -fclose_error_out: - if (fclose(file) < 0) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("failed to close tty pid file %s: %s"), - vm->ttyPidFile, strerror(errno)); - } - -error_out: - return rc; -} - -/** - * lxcLoadTtyPid: - * @driver: pointer to driver - * @vm: Ptr to VM - * - * Loads the pid of the tty forward process from the pid file. - * LOCAL_STATE_DIR/run/libvirt/lxc/{container_name}.pid - * - * Returns - * > 0 - pid of tty process - * 0 - no tty pid file - * -1 - error - */ -int lxcLoadTtyPid(const lxc_driver_t *driver, lxc_vm_t *vm) -{ - int rc = -1; - FILE *file; - - if (vm->ttyPidFile[0] == 0x00) { - if ((rc = virFileMakePath(driver->stateDir))) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot create lxc state directory %s: %s"), - driver->stateDir, strerror(rc)); - goto cleanup; - } - - if (virFileBuildPath(driver->stateDir, vm->def->name, ".pid", - vm->ttyPidFile, PATH_MAX) < 0) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot construct tty pid file path")); - goto cleanup; - } - } - - if (!(file = fopen(vm->ttyPidFile, "r"))) { - if (ENOENT == errno) { - rc = 0; - goto cleanup; - } - - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot open tty pid file %s: %s"), - vm->ttyPidFile, strerror(errno)); - goto cleanup; - } - - if (fscanf(file, "%d", &(vm->pid)) < 0) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot read tty pid file %s: %s"), - vm->ttyPidFile, strerror(errno)); - goto cleanup; - } - - if (fclose(file) < 0) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("failed to close tty pid file %s: %s"), - vm->ttyPidFile, strerror(errno)); - goto cleanup; - } - - rc = vm->pid; - - cleanup: - return rc; -} - -/** - * lxcDeleteTtyPid: - * @vm: Ptr to VM - * - * Unlinks the tty pid file for the vm - * LOCAL_STATE_DIR/run/libvirt/lxc/{container_name}.pid - * - * Returns on 0 success or -1 in case of error - */ -int lxcDeleteTtyPidFile(const lxc_vm_t *vm) -{ - if (vm->ttyPidFile[0] == 0x00) { - goto no_file; - } - - if (unlink(vm->ttyPidFile) < 0) { - if (errno == ENOENT) { - goto no_file; - } - - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot remove ttyPidFile %s: %s"), vm->ttyPidFile, - strerror(errno)); - return -1; - } - -no_file: - return 0; -} - #endif /* WITH_LXC */ diff -r 8093fb566748 src/lxc_conf.h --- a/src/lxc_conf.h Fri Aug 01 14:47:33 2008 +0100 +++ b/src/lxc_conf.h Tue Aug 05 12:13:24 2008 +0100 @@ -46,7 +46,6 @@ struct __lxc_net_def { int type; char *parentVeth; /* veth device in parent namespace */ - char *containerVeth; /* veth device in container namespace */ char *txName; /* bridge or network name */ lxc_net_def_t *next; @@ -87,11 +86,10 @@ struct __lxc_vm { int pid; int state; + int monitor; char configFile[PATH_MAX]; char configFileBase[PATH_MAX]; - - char ttyPidFile[PATH_MAX]; lxc_vm_def_t *def; @@ -103,8 +101,9 @@ lxc_vm_t *vms; int nactivevms; int ninactivevms; - char* configDir; - char* stateDir; + char *configDir; + char *stateDir; + char *logDir; int have_netns; }; @@ -154,9 +153,6 @@ lxc_driver_t *driver, const char *configFile, const char *name); -int lxcStoreTtyPid(const lxc_driver_t *driver, lxc_vm_t *vm); -int lxcLoadTtyPid(const lxc_driver_t *driver, lxc_vm_t *vm); -int lxcDeleteTtyPidFile(const lxc_vm_t *vm); void lxcError(virConnectPtr conn, virDomainPtr dom, diff -r 8093fb566748 src/lxc_container.c --- a/src/lxc_container.c Fri Aug 01 14:47:33 2008 +0100 +++ b/src/lxc_container.c Tue Aug 05 12:13:24 2008 +0100 @@ -69,6 +69,8 @@ typedef struct __lxc_child_argv lxc_child_argv_t; struct __lxc_child_argv { lxc_vm_def_t *config; + int nveths; + char **veths; int monitor; char *ttyPath; }; @@ -171,8 +173,7 @@ * * Returns 0 on success or -1 in case of error */ -int lxcContainerSendContinue(virConnectPtr conn, - int control) +int lxcContainerSendContinue(int control) { int rc = -1; lxc_message_t msg = LXC_CONTINUE_MSG; @@ -180,7 +181,7 @@ writeCount = safewrite(control, &msg, sizeof(msg)); if (writeCount != sizeof(msg)) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, _("unable to send container continue message: %s"), strerror(errno)); goto error_out; @@ -230,21 +231,21 @@ * * Returns 0 on success or nonzero in case of error */ -static int lxcContainerEnableInterfaces(const lxc_vm_def_t *def) +static int lxcContainerEnableInterfaces(int nveths, + char **veths) { - int rc = 0; - const lxc_net_def_t *net; + int rc = 0, i; - for (net = def->nets; net; net = net->next) { - DEBUG("Enabling %s", net->containerVeth); - rc = vethInterfaceUpOrDown(net->containerVeth, 1); + for (i = 0 ; i < nveths ; i++) { + DEBUG("Enabling %s", veths[i]); + rc = vethInterfaceUpOrDown(veths[i], 1); if (0 != rc) { goto error_out; } } /* enable lo device only if there were other net devices */ - if (def->nets) + if (veths) rc = vethInterfaceUpOrDown("lo", 1); error_out: @@ -311,7 +312,7 @@ return -1; /* enable interfaces */ - if (lxcContainerEnableInterfaces(vmDef) < 0) + if (lxcContainerEnableInterfaces(argv->nveths, argv->veths) < 0) return -1; /* this function will only return if an error occured */ @@ -320,7 +321,6 @@ /** * lxcContainerStart: - * @conn: pointer to connection * @driver: pointer to driver structure * @vm: pointer to virtual machine structure * @@ -328,8 +328,9 @@ * * Returns PID of container on success or -1 in case of error */ -int lxcContainerStart(virConnectPtr conn, - lxc_vm_def_t *def, +int lxcContainerStart(lxc_vm_def_t *def, + int nveths, + char **veths, int control, char *ttyPath) { @@ -337,12 +338,11 @@ int flags; int stacksize = getpagesize() * 4; char *stack, *stacktop; - lxc_child_argv_t args = { def, control, ttyPath }; + lxc_child_argv_t args = { def, nveths, veths, control, ttyPath }; /* allocate a stack for the container */ if (VIR_ALLOC_N(stack, stacksize) < 0) { - lxcError(conn, NULL, VIR_ERR_NO_MEMORY, - _("unable to allocate container stack")); + lxcError(NULL, NULL, VIR_ERR_NO_MEMORY, NULL); return -1; } stacktop = stack + stacksize; @@ -357,7 +357,7 @@ DEBUG("clone() returned, %d", pid); if (pid < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, _("clone() failed, %s"), strerror(errno)); return -1; } @@ -379,8 +379,9 @@ char *stack; int childStatus; - if (features & LXC_CONTAINER_FEATURE_NET) + if (features & LXC_CONTAINER_FEATURE_NET) { flags |= CLONE_NEWNET; + } if (VIR_ALLOC_N(stack, getpagesize() * 4) < 0) { DEBUG0("Unable to allocate stack"); diff -r 8093fb566748 src/lxc_container.h --- a/src/lxc_container.h Fri Aug 01 14:47:33 2008 +0100 +++ b/src/lxc_container.h Tue Aug 05 12:13:24 2008 +0100 @@ -32,11 +32,11 @@ LXC_CONTAINER_FEATURE_NET = (1 << 0), }; -int lxcContainerSendContinue(virConnectPtr conn, - int control); +int lxcContainerSendContinue(int control); -int lxcContainerStart(virConnectPtr conn, - lxc_vm_def_t *def, +int lxcContainerStart(lxc_vm_def_t *def, + int nveths, + char **veths, int control, char *ttyPath); diff -r 8093fb566748 src/lxc_controller.c --- a/src/lxc_controller.c Fri Aug 01 14:47:33 2008 +0100 +++ b/src/lxc_controller.c Tue Aug 05 12:13:24 2008 +0100 @@ -26,16 +26,59 @@ #ifdef WITH_LXC #include <sys/epoll.h> +#include <sys/wait.h> +#include <sys/socket.h> #include <unistd.h> +#include <paths.h> +#include <fcntl.h> #include "internal.h" #include "util.h" #include "lxc_conf.h" +#include "lxc_container.h" #include "lxc_controller.h" +#include "veth.h" +#include "memory.h" +#include "util.h" #define DEBUG(fmt,...) VIR_DEBUG(__FILE__, fmt, __VA_ARGS__) +#define DEBUG0(msg) VIR_DEBUG(__FILE__, "%s", msg) + + +#define LXC_HANDSHAKE_REQUEST 'm' +#define LXC_HANDSHAKE_REPLY 'r' + +int lxcControllerClientHandshake(int monitor) +{ + char c; + if (saferead(monitor, &c, sizeof(c)) != sizeof(c)) + return -1; + if (c != LXC_HANDSHAKE_REQUEST) { + errno = EINVAL; + return -1; + } + c = LXC_HANDSHAKE_REPLY; + if (safewrite(monitor, &c, sizeof(c)) != sizeof(c)) + return -1; + return 0; +} + +static int lxcControllerServerHandshake(int monitor) +{ + char c = LXC_HANDSHAKE_REQUEST; + if (safewrite(monitor, &c, sizeof(c)) != sizeof(c)) + return -1; + if (saferead(monitor, &c, sizeof(c)) != sizeof(c)) + return -1; + if (c != LXC_HANDSHAKE_REPLY) { + errno = EINVAL; + return -1; + } + return 0; +} + /** * lxcFdForward: @@ -91,7 +134,10 @@ * * Returns 0 on success or -1 in case of error */ -int lxcControllerMain(int appPty, int contPty) +static int lxcControllerMain(int monitor, + int client, + int appPty, + int contPty) { int rc = -1; int epollFd; @@ -120,15 +166,29 @@ memset(&epollEvent, 0x00, sizeof(epollEvent)); epollEvent.events = EPOLLIN|EPOLLET; /* edge triggered */ epollEvent.data.fd = appPty; - epollEvent.data.u32 = 0; /* fdArray position */ if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, appPty, &epollEvent)) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, _("epoll_ctl(appPty) failed: %s"), strerror(errno)); goto cleanup; } epollEvent.data.fd = contPty; - epollEvent.data.u32 = 1; /* fdArray position */ if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, contPty, &epollEvent)) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("epoll_ctl(contPty) failed: %s"), strerror(errno)); + goto cleanup; + } + + epollEvent.events = EPOLLIN; + epollEvent.data.fd = monitor; + if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, monitor, &epollEvent)) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("epoll_ctl(contPty) failed: %s"), strerror(errno)); + goto cleanup; + } + + epollEvent.events = EPOLLHUP; + epollEvent.data.fd = client; + if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, client, &epollEvent)) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, _("epoll_ctl(contPty) failed: %s"), strerror(errno)); goto cleanup; @@ -138,23 +198,46 @@ /* if active fd's, return if no events, else wait forever */ timeout = (numActive > 0) ? 0 : -1; numEvents = epoll_wait(epollFd, &epollEvent, 1, timeout); - if (0 < numEvents) { - if (epollEvent.events & EPOLLIN) { - curFdOff = epollEvent.data.u32; - if (!fdArray[curFdOff].active) { - fdArray[curFdOff].active = 1; - ++numActive; + if (numEvents > 0) { + if (epollEvent.data.fd == monitor) { + int fd = accept(monitor, NULL, 0); + if (client != -1 || /* Already connected, so kick new one out */ + lxcControllerServerHandshake(client) < 0) { + close(fd); + continue; } - - } else if (epollEvent.events & EPOLLHUP) { - DEBUG("EPOLLHUP from fd %d", epollEvent.data.fd); - continue; + client = fd; + epollEvent.events = EPOLLHUP; + epollEvent.data.fd = client; + if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, client, &epollEvent)) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("epoll_ctl(contPty) failed: %s"), strerror(errno)); + goto cleanup; + } + } else if (client != -1 && epollEvent.data.fd == client) { + if (0 > epoll_ctl(epollFd, EPOLL_CTL_DEL, client, &epollEvent)) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("epoll_ctl(contPty) failed: %s"), strerror(errno)); + goto cleanup; + } + close(client); + client = -1; } else { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("error event %d"), epollEvent.events); - goto cleanup; + if (epollEvent.events & EPOLLIN) { + curFdOff = epollEvent.data.fd == appPty ? 0 : 1; + if (!fdArray[curFdOff].active) { + fdArray[curFdOff].active = 1; + ++numActive; + } + } else if (epollEvent.events & EPOLLHUP) { + DEBUG("EPOLLHUP from fd %d", epollEvent.data.fd); + continue; + } else { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("error event %d"), epollEvent.events); + goto cleanup; + } } - } else if (0 == numEvents) { if (2 == numActive) { /* both fds active, toggle between the two */ @@ -202,4 +285,236 @@ return rc; } + + +/** + * lxcControllerMoveInterfaces + * @nveths: number of interfaces + * @veths: interface names + * @container: pid of container + * + * Moves network interfaces into a container's namespace + * + * Returns 0 on success or -1 in case of error + */ +static int lxcControllerMoveInterfaces(int nveths, + char **veths, + pid_t container) +{ + int i; + for (i = 0 ; i < nveths ; i++) + if (moveInterfaceToNetNs(veths[i], container) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to move interface %s to ns %d"), + veths[i], container); + return -1; + } + + return 0; +} + + +/** + * lxcCleanupInterfaces: + * @conn: pointer to connection + * @vm: pointer to virtual machine structure + * + * Cleans up the container interfaces by deleting the veth device pairs. + * + * Returns 0 on success or -1 in case of error + */ +static int lxcControllerCleanupInterfaces(int nveths, + char **veths) +{ + int i; + for (i = 0 ; i < nveths ; i++) + if (vethDelete(veths[i]) < 0) + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to delete veth: %s"), veths[i]); + /* will continue to try to cleanup any other interfaces */ + + return 0; +} + + +static int +lxcControllerRun(const char *stateDir, + lxc_vm_def_t *def, + int nveths, + char **veths, + int monitor, + int client, + int appPty) +{ + int rc = -1; + int control[2] = { -1, -1}; + int containerPty; + char *containerPtyPath; + pid_t container = -1; + + if (socketpair(PF_UNIX, SOCK_STREAM, 0, control) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("sockpair failed: %s"), strerror(errno)); + goto cleanup; + } + + if (virFileOpenTty(&containerPty, + &containerPtyPath, + 0) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to allocate tty: %s"), strerror(errno)); + goto cleanup; + } + + if ((container = lxcContainerStart(def, + nveths, + veths, + control[1], + containerPtyPath)) < 0) + goto cleanup; + close(control[1]); + control[1] = -1; + + if (lxcControllerMoveInterfaces(nveths, veths, container) < 0) + goto cleanup; + + if (lxcContainerSendContinue(control[0]) < 0) + goto cleanup; + + rc = lxcControllerMain(monitor, client, appPty, containerPty); + +cleanup: + if (control[0] != -1) + close(control[0]); + if (control[1] != -1) + close(control[1]); + VIR_FREE(containerPtyPath); + if (containerPty != -1) + close(containerPty); + + kill(container, SIGTERM); + waitpid(container, NULL, 0); + lxcControllerCleanupInterfaces(nveths, veths); + virFileDeletePid(stateDir, def->name); + return -1; +} + + +int lxcControllerStart(const char *stateDir, + lxc_vm_def_t *def, + int nveths, + char **veths, + int monitor, + int appPty, + int logfd) +{ + pid_t pid; + int rc; + int status, null; + int open_max, i; + int client; + + if ((pid = fork()) < 0) + return -1; + + if (pid > 0) { + /* Original caller waits for first child to exit */ + while (1) { + rc = waitpid(pid, &status, 0); + if (rc < 0) { + if (errno == EINTR) + continue; + return -1; + } + if (rc != pid) { + fprintf(stderr, + _("Unexpected pid %d != %d from waitpid\n"), + rc, pid); + return -1; + } + if (WIFEXITED(status) && + WEXITSTATUS(status) == 0) + return 0; + else { + fprintf(stderr, + _("Unexpected status %d from pid %d\n"), + status, pid); + return -1; + } + } + } + + /* First child is running here */ + + if (chdir("/") < 0) { + fprintf(stderr, _("Unable to change to root dir: %s\n"), + strerror(errno)); + _exit(-1); + } + + if (setsid() < 0) { + fprintf(stderr, _("Unable to become session leader: %s\n"), + strerror(errno)); + _exit(-1); + } + + if ((null = open(_PATH_DEVNULL, O_RDONLY)) < 0) { + fprintf(stderr, _("Unable to open %s: %s\n"), + _PATH_DEVNULL, strerror(errno)); + _exit(-1); + } + + open_max = sysconf (_SC_OPEN_MAX); + for (i = 0; i < open_max; i++) + if (i != appPty && + i != monitor && + i != logfd && + i != null) + close(i); + + if (dup2(null, STDIN_FILENO) < 0 || + dup2(logfd, STDOUT_FILENO) < 0 || + dup2(logfd, STDERR_FILENO) < 0) { + fprintf(stderr, _("Unable to redirect stdio: %s\n"), + strerror(errno)); + _exit(-1); + } + + close(null); + close(logfd); + + /* Now fork the real controller process */ + if ((pid = fork()) < 0) { + fprintf(stderr, _("Unable to fork controller: %s\n"), + strerror(errno)); + _exit(-1); + } + + if (pid > 0) { + /* First child now exits */ + _exit(0); + } + + /* This is real controller running... */ + if ((rc = virFileWritePid(stateDir, def->name, getpid())) != 0) { + fprintf(stderr, _("Unable to write pid file: %s\n"), + strerror(rc)); + _exit(-1); + } + + /* Accept initial client which is the libvirtd daemon */ + if ((client = accept(monitor, NULL, 0)) < 0 || + lxcControllerServerHandshake(client) < 0) { + fprintf(stderr, _("Failed connection from LXC driver: %s\n"), + strerror(errno)); + _exit(-1); + } + + /* Controlling libvirtd LXC driver now knows + what our PID is, and is able to cleanup after + us from now on */ + _exit(lxcControllerRun(stateDir, def, nveths, veths, monitor, client, appPty)); +} + + #endif diff -r 8093fb566748 src/lxc_controller.h --- a/src/lxc_controller.h Fri Aug 01 14:47:33 2008 +0100 +++ b/src/lxc_controller.h Tue Aug 05 12:13:24 2008 +0100 @@ -26,7 +26,17 @@ #ifdef WITH_LXC -int lxcControllerMain(int appPty, int contPty); +#include "lxc_conf.h" + +int lxcControllerClientHandshake(int monitor); + +int lxcControllerStart(const char *stateDir, + lxc_vm_def_t *def, + int nveths, + char **veths, + int monitor, + int appPty, + int logfd); #endif /* WITH_LXC */ diff -r 8093fb566748 src/lxc_driver.c --- a/src/lxc_driver.c Fri Aug 01 14:47:33 2008 +0100 +++ b/src/lxc_driver.c Tue Aug 05 12:13:24 2008 +0100 @@ -31,22 +31,23 @@ #include <stdbool.h> #include <string.h> #include <sys/types.h> -#include <termios.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <sys/poll.h> #include <unistd.h> #include <wait.h> +#include "internal.h" #include "lxc_conf.h" #include "lxc_container.h" #include "lxc_driver.h" #include "lxc_controller.h" -#include "driver.h" -#include "internal.h" #include "memory.h" #include "util.h" -#include "memory.h" #include "bridge.h" -#include "qemu_conf.h" #include "veth.h" +#include "event.h" + /* debug macros */ #define DEBUG(fmt,...) VIR_DEBUG(__FILE__, fmt, __VA_ARGS__) @@ -284,8 +285,6 @@ vm->configFile[0] = '\0'; - lxcDeleteTtyPidFile(vm); - lxcRemoveInactiveVM(driver, vm); return 0; @@ -339,10 +338,59 @@ return lxcGenerateXML(dom->conn, driver, vm, vm->def); } + +/** + * lxcVmCleanup: + * @vm: Ptr to VM to clean up + * + * waitpid() on the container process. kill and wait the tty process + * This is called by boh lxcDomainDestroy and lxcSigHandler when a + * container exits. + * + * Returns 0 on success or -1 in case of error + */ +static int lxcVMCleanup(virConnectPtr conn, + lxc_driver_t *driver, + lxc_vm_t * vm) +{ + int rc = -1; + int waitRc; + int childStatus = -1; + + while (((waitRc = waitpid(vm->pid, &childStatus, 0)) == -1) && + errno == EINTR); + + if ((waitRc != vm->pid) && (errno != ECHILD)) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("waitpid failed to wait for container %d: %d %s"), + vm->pid, waitRc, strerror(errno)); + } + + rc = 0; + + if (WIFEXITED(childStatus)) { + rc = WEXITSTATUS(childStatus); + DEBUG("container exited with rc: %d", rc); + } + + virEventRemoveHandle(vm->monitor); + close(vm->monitor); + + virFileDeletePid(driver->stateDir, vm->def->name); + + vm->state = VIR_DOMAIN_SHUTOFF; + vm->pid = -1; + vm->def->id = -1; + vm->monitor = -1; + driver->nactivevms--; + driver->ninactivevms++; + + return rc; +} + /** * lxcSetupInterfaces: - * @conn: pointer to connection - * @vm: pointer to virtual machine structure + * @def: pointer to virtual machine structure * * Sets up the container interfaces by creating the veth device pairs and * attaching the parent end to the appropriate bridge. The container end @@ -351,24 +399,21 @@ * Returns 0 on success or -1 in case of error */ static int lxcSetupInterfaces(virConnectPtr conn, - lxc_vm_t *vm) + lxc_vm_def_t *def, + int *nveths, + char ***veths) { int rc = -1; - lxc_driver_t *driver = conn->privateData; - struct qemud_driver *networkDriver = - (struct qemud_driver *)(conn->networkPrivateData); - lxc_net_def_t *net = vm->def->nets; - char* bridge; + lxc_net_def_t *net; + char *bridge = NULL; char parentVeth[PATH_MAX] = ""; char containerVeth[PATH_MAX] = ""; + brControl *brctl = NULL; - if ((vm->def->nets != NULL) && (driver->have_netns == 0)) { - lxcError(conn, NULL, VIR_ERR_NO_SUPPORT, - _("System lacks NETNS support")); + if (brInit(&brctl) != 0) return -1; - } - for (net = vm->def->nets; net; net = net->next) { + for (net = def->nets; net; net = net->next) { if (LXC_NET_NETWORK == net->type) { virNetworkPtr network = virNetworkLookupByName(conn, net->txName); if (!network) { @@ -378,7 +423,6 @@ bridge = virNetworkGetBridgeName(network); virNetworkFree(network); - } else { bridge = net->txName; } @@ -394,9 +438,6 @@ if (NULL != net->parentVeth) { strcpy(parentVeth, net->parentVeth); } - if (NULL != net->containerVeth) { - strcpy(containerVeth, net->containerVeth); - } DEBUG("parentVeth: %s, containerVeth: %s", parentVeth, containerVeth); if (0 != (rc = vethCreate(parentVeth, PATH_MAX, containerVeth, PATH_MAX))) { lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, @@ -406,24 +447,18 @@ if (NULL == net->parentVeth) { net->parentVeth = strdup(parentVeth); } - if (NULL == net->containerVeth) { - net->containerVeth = strdup(containerVeth); - } + if (VIR_REALLOC_N(*veths, (*nveths)+1) < 0) + goto error_exit; + if (((*veths)[(*nveths)++] = strdup(containerVeth)) == NULL) + goto error_exit; - if ((NULL == net->parentVeth) || (NULL == net->containerVeth)) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + if (NULL == net->parentVeth) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, _("failed to allocate veth names")); goto error_exit; } - if (!(networkDriver->brctl) && (rc = brInit(&(networkDriver->brctl)))) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - _("cannot initialize bridge support: %s"), - strerror(rc)); - goto error_exit; - } - - if (0 != (rc = brAddInterface(networkDriver->brctl, bridge, parentVeth))) { + if (0 != (rc = brAddInterface(brctl, bridge, parentVeth))) { lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, _("failed to add %s device to %s: %s"), parentVeth, @@ -433,7 +468,7 @@ } if (0 != (rc = vethInterfaceUpOrDown(parentVeth, 1))) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, _("failed to enable parent ns veth device: %d"), rc); goto error_exit; } @@ -443,136 +478,151 @@ rc = 0; error_exit: + brShutdown(brctl); return rc; } -/** - * lxcMoveInterfacesToNetNs: - * @conn: pointer to connection - * @vm: pointer to virtual machine structure - * - * Starts a container process by calling clone() with the namespace flags - * - * Returns 0 on success or -1 in case of error - */ -static int lxcMoveInterfacesToNetNs(virConnectPtr conn, - const lxc_vm_t *vm) +static int lxcMonitorServer(virConnectPtr conn, + lxc_driver_t * driver, + lxc_vm_t *vm) { - int rc = -1; - lxc_net_def_t *net; + char *sockpath = NULL; + int fd; + struct sockaddr_un addr; - for (net = vm->def->nets; net; net = net->next) { - if (0 != moveInterfaceToNetNs(net->containerVeth, vm->def->id)) { + if (asprintf(&sockpath, "%s/%s.sock", + driver->stateDir, vm->def->name) < 0) { + lxcError(conn, NULL, VIR_ERR_NO_MEMORY, NULL); + return -1; + } + + if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to create server socket: %s"), + strerror(errno)); + goto error; + } + + unlink(sockpath); + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, sockpath, sizeof(addr.sun_path)); + + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to bind server socket: %s"), + strerror(errno)); + goto error; + } + if (listen(fd, 30 /* backlog */ ) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to listen server socket: %s"), + strerror(errno)); + goto error; + return (-1); + } + + VIR_FREE(sockpath); + return fd; + +error: + VIR_FREE(sockpath); + if (fd != -1) + close(fd); + return -1; +} + +static int lxcMonitorClient(virConnectPtr conn, + lxc_driver_t * driver, + lxc_vm_t *vm) +{ + char *sockpath = NULL; + int fd; + struct sockaddr_un addr; + + if (asprintf(&sockpath, "%s/%s.sock", + driver->stateDir, vm->def->name) < 0) { + lxcError(conn, NULL, VIR_ERR_NO_MEMORY, NULL); + return -1; + } + + if ((fd = socket(PF_UNIX, SOCK_STREAM, 0)) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to create client socket: %s"), + strerror(errno)); + goto error; + } + + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, sockpath, sizeof(addr.sun_path)); + + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to connect to client socket: %s"), + strerror(errno)); + goto error; + } + + if (lxcControllerClientHandshake(fd) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to handshake with client: %s"), + strerror(errno)); + goto error; + } + + VIR_FREE(sockpath); + return fd; + +error: + VIR_FREE(sockpath); + if (fd != -1) + close(fd); + return -1; +} + + +static int lxcVmTerminate(virConnectPtr conn, + lxc_driver_t *driver, + lxc_vm_t *vm, + int signum) +{ + if (signum == 0) + signum = SIGINT; + + if (kill(vm->pid, signum) < 0) { + if (errno != ESRCH) { lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - _("failed to move interface %s to ns %d"), - net->containerVeth, vm->def->id); - goto error_exit; + _("failed to kill pid %d: %s"), + vm->pid, strerror(errno)); + return -1; } } - rc = 0; + vm->state = VIR_DOMAIN_SHUTDOWN; -error_exit: - return rc; + return lxcVMCleanup(conn, driver, vm); } -/** - * lxcCleanupInterfaces: - * @conn: pointer to connection - * @vm: pointer to virtual machine structure - * - * Cleans up the container interfaces by deleting the veth device pairs. - * - * Returns 0 on success or -1 in case of error - */ -static int lxcCleanupInterfaces(const lxc_vm_t *vm) +static void lxcMonitorEvent(int fd, + int events ATTRIBUTE_UNUSED, + void *data) { - int rc = -1; - lxc_net_def_t *net; + lxc_driver_t *driver = data; + lxc_vm_t *vm = driver->vms; - for (net = vm->def->nets; net; net = net->next) { - if (0 != (rc = vethDelete(net->parentVeth))) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("failed to delete veth: %s"), net->parentVeth); - /* will continue to try to cleanup any other interfaces */ - } + while (vm) { + if (vm->monitor == fd) + break; + vm = vm->next; + } + if (!vm) { + virEventRemoveHandle(fd); + return; } - return 0; -} - - -/** - * lxcOpenTty: - * @conn: pointer to connection - * @ttymaster: pointer to int. On success, set to fd for master end - * @ttyName: On success, will point to string slave end of tty. Caller - * must free when done (such as in lxcFreeVM). - * - * Opens and configures container tty. - * - * Returns 0 on success or -1 in case of error - */ -static int lxcOpenTty(virConnectPtr conn, - int *ttymaster, - char **ttyName, - int rawmode) -{ - int rc = -1; - - *ttymaster = posix_openpt(O_RDWR|O_NOCTTY|O_NONBLOCK); - if (*ttymaster < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - _("posix_openpt failed: %s"), strerror(errno)); - goto cleanup; - } - - if (unlockpt(*ttymaster) < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - _("unlockpt failed: %s"), strerror(errno)); - goto cleanup; - } - - if (rawmode) { - struct termios ttyAttr; - if (tcgetattr(*ttymaster, &ttyAttr) < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - "tcgetattr() failed: %s", strerror(errno)); - goto cleanup; - } - - cfmakeraw(&ttyAttr); - - if (tcsetattr(*ttymaster, TCSADRAIN, &ttyAttr) < 0) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - "tcsetattr failed: %s", strerror(errno)); - goto cleanup; - } - } - - if (ttyName) { - char tempTtyName[PATH_MAX]; - if (0 != ptsname_r(*ttymaster, tempTtyName, sizeof(tempTtyName))) { - lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - _("ptsname_r failed: %s"), strerror(errno)); - goto cleanup; - } - - if ((*ttyName = strdup(tempTtyName)) == NULL) { - lxcError(conn, NULL, VIR_ERR_NO_MEMORY, NULL); - goto cleanup; - } - } - - rc = 0; - -cleanup: - if (rc != 0 && - *ttymaster != -1) { - close(*ttymaster); - } - - return rc; + if (lxcVmTerminate(NULL, driver, vm, SIGINT) < 0) + virEventRemoveHandle(fd); } @@ -590,81 +640,106 @@ lxc_driver_t * driver, lxc_vm_t * vm) { - int rc = -1; - int sockpair[2]; - int containerTty, parentTty; - char *containerTtyPath = NULL; + int rc = -1, i; + int monitor; + int parentTty; + char *logfile = NULL; + int logfd = -1; + int nveths = 0; + char **veths = NULL; + + if (virFileMakePath(driver->logDir) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("cannot create log directory %s: %s"), + driver->logDir, strerror(rc)); + return -1; + } + + if (asprintf(&logfile, "%s/%s.log", + driver->logDir, vm->def->name) < 0) { + lxcError(conn, NULL, VIR_ERR_NO_MEMORY, NULL); + return -1; + } + + if ((monitor = lxcMonitorServer(conn, driver, vm)) < 0) + goto cleanup; /* open parent tty */ VIR_FREE(vm->def->tty); - if (lxcOpenTty(conn, &parentTty, &vm->def->tty, 1) < 0) { - goto cleanup; - } - - /* open container tty */ - if (lxcOpenTty(conn, &containerTty, &containerTtyPath, 0) < 0) { - goto cleanup; - } - - /* fork process to handle the tty io forwarding */ - if ((vm->pid = fork()) < 0) { + if (virFileOpenTty(&parentTty, &vm->def->tty, 1) < 0) { lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - _("unable to fork tty forwarding process: %s"), + _("failed to allocate tty: %s"), strerror(errno)); goto cleanup; } - if (vm->pid == 0) { - /* child process calls forward routine */ - lxcControllerMain(parentTty, containerTty); - } + if (lxcSetupInterfaces(conn, vm->def, &nveths, &veths) != 0) + goto cleanup; - if (lxcStoreTtyPid(driver, vm)) { - DEBUG0("unable to store tty pid"); - } - - close(parentTty); - close(containerTty); - - if (0 != (rc = lxcSetupInterfaces(conn, vm))) { + if ((logfd = open(logfile, O_WRONLY | O_TRUNC | O_CREAT, + S_IRUSR|S_IWUSR)) < 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to open %s: %s"), logfile, + strerror(errno)); goto cleanup; } - /* create a socket pair to send continue message to the container once */ - /* we've completed the post clone configuration */ - if (0 != socketpair(PF_UNIX, SOCK_STREAM, 0, sockpair)) { + if (lxcControllerStart(driver->stateDir, + vm->def, nveths, veths, + monitor, parentTty, logfd) < 0) + goto cleanup; + /* Close the server side of the monitor, now owned + * by the controller process */ + close(monitor); + monitor = -1; + + /* Connect to the controller as a client *first* because + * this will block until the child has written their + * pid file out to disk */ + if ((vm->monitor = lxcMonitorClient(conn, driver, vm)) < 0) + goto cleanup; + + /* And get its pid */ + if ((rc = virFileReadPid(driver->stateDir, vm->def->name, &vm->pid)) != 0) { lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, - _("sockpair failed: %s"), strerror(errno)); + _("Failed to read pid file %s/%s.pid: %s"), + driver->stateDir, vm->def->name, strerror(errno)); + rc = -1; goto cleanup; } - /* check this rc */ - - vm->def->id = lxcContainerStart(conn, - vm->def, - sockpair[1], - containerTtyPath); - if (vm->def->id == -1) - goto cleanup; - lxcSaveConfig(conn, driver, vm, vm->def); - - rc = lxcMoveInterfacesToNetNs(conn, vm); - if (rc != 0) - goto cleanup; - - rc = lxcContainerSendContinue(conn, sockpair[0]); - if (rc != 0) - goto cleanup; - + vm->def->id = vm->pid; vm->state = VIR_DOMAIN_RUNNING; driver->ninactivevms--; driver->nactivevms++; + if (virEventAddHandle(vm->monitor, + POLLERR | POLLHUP, + lxcMonitorEvent, + driver) < 0) { + lxcVmTerminate(conn, driver, vm, 0); + goto cleanup; + } + + rc = 0; + cleanup: - close(sockpair[0]); - close(sockpair[1]); - VIR_FREE(containerTtyPath); - + for (i = 0 ; i < nveths ; i++) { + if (rc != 0) + vethDelete(veths[i]); + VIR_FREE(veths[i]); + } + if (monitor != -1) + close(monitor); + if (rc != 0 && vm->monitor != -1) { + close(vm->monitor); + vm->monitor = -1; + } + if (parentTty != -1) + close(parentTty); + if (logfd != -1) + close(logfd); + VIR_FREE(logfile); return rc; } @@ -752,105 +827,18 @@ */ static int lxcDomainShutdown(virDomainPtr dom) { - int rc = -1; lxc_driver_t *driver = (lxc_driver_t*)dom->conn->privateData; lxc_vm_t *vm = lxcFindVMByID(driver, dom->id); if (!vm) { lxcError(dom->conn, dom, VIR_ERR_INVALID_DOMAIN, _("no domain with id %d"), dom->id); - goto error_out; + return -1; } - if (0 > (kill(vm->def->id, SIGINT))) { - if (ESRCH != errno) { - lxcError(dom->conn, dom, VIR_ERR_INTERNAL_ERROR, - _("sending SIGTERM failed: %s"), strerror(errno)); - - goto error_out; - } - } - - vm->state = VIR_DOMAIN_SHUTDOWN; - - rc = 0; - -error_out: - return rc; + return lxcVmTerminate(dom->conn, driver, vm, 0); } -/** - * lxcVmCleanup: - * @vm: Ptr to VM to clean up - * - * waitpid() on the container process. kill and wait the tty process - * This is called by boh lxcDomainDestroy and lxcSigHandler when a - * container exits. - * - * Returns 0 on success or -1 in case of error - */ -static int lxcVMCleanup(lxc_driver_t *driver, lxc_vm_t * vm) -{ - int rc = -1; - int waitRc; - int childStatus = -1; - - /* if this fails, we'll continue. it will report any errors */ - lxcCleanupInterfaces(vm); - - while (((waitRc = waitpid(vm->def->id, &childStatus, 0)) == -1) && - errno == EINTR); - - if ((waitRc != vm->def->id) && (errno != ECHILD)) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("waitpid failed to wait for container %d: %d %s"), - vm->def->id, waitRc, strerror(errno)); - goto kill_tty; - } - - rc = 0; - - if (WIFEXITED(childStatus)) { - rc = WEXITSTATUS(childStatus); - DEBUG("container exited with rc: %d", rc); - } - -kill_tty: - if (2 > vm->pid) { - DEBUG("not killing tty process with pid %d", vm->pid); - goto tty_error_out; - } - - if (0 > (kill(vm->pid, SIGKILL))) { - if (ESRCH != errno) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("sending SIGKILL to tty process failed: %s"), - strerror(errno)); - - goto tty_error_out; - } - } - - while (((waitRc = waitpid(vm->pid, &childStatus, 0)) == -1) && - errno == EINTR); - - if ((waitRc != vm->pid) && (errno != ECHILD)) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("waitpid failed to wait for tty %d: %d %s"), - vm->pid, waitRc, strerror(errno)); - } - -tty_error_out: - vm->state = VIR_DOMAIN_SHUTOFF; - vm->pid = -1; - lxcDeleteTtyPidFile(vm); - vm->def->id = -1; - driver->nactivevms--; - driver->ninactivevms++; - lxcSaveConfig(NULL, driver, vm, vm->def); - - return rc; - } /** * lxcDomainDestroy: @@ -862,31 +850,16 @@ */ static int lxcDomainDestroy(virDomainPtr dom) { - int rc = -1; lxc_driver_t *driver = (lxc_driver_t*)dom->conn->privateData; lxc_vm_t *vm = lxcFindVMByID(driver, dom->id); if (!vm) { lxcError(dom->conn, dom, VIR_ERR_INVALID_DOMAIN, _("no domain with id %d"), dom->id); - goto error_out; + return -1; } - if (0 > (kill(vm->def->id, SIGKILL))) { - if (ESRCH != errno) { - lxcError(dom->conn, dom, VIR_ERR_INTERNAL_ERROR, - _("sending SIGKILL failed: %s"), strerror(errno)); - - goto error_out; - } - } - - vm->state = VIR_DOMAIN_SHUTDOWN; - - rc = lxcVMCleanup(driver, vm); - -error_out: - return rc; + return lxcVmTerminate(dom->conn, driver, vm, SIGKILL); } static int lxcCheckNetNsSupport(void) @@ -907,6 +880,7 @@ static int lxcStartup(void) { uid_t uid = getuid(); + lxc_vm_t *vm; /* Check that the user is root */ if (0 != uid) { @@ -935,6 +909,36 @@ return -1; } + vm = lxc_driver->vms; + while (vm) { + int rc; + if ((vm->monitor = lxcMonitorClient(NULL, lxc_driver, vm)) < 0) { + vm = vm->next; + continue; + } + + /* Read pid from controller */ + if ((rc = virFileReadPid(lxc_driver->stateDir, vm->def->name, &vm->pid)) != 0) { + close(vm->monitor); + vm->monitor = -1; + vm = vm->next; + continue; + } + + if (vm->pid != 0) { + vm->def->id = vm->pid; + vm->state = VIR_DOMAIN_RUNNING; + lxc_driver->ninactivevms--; + lxc_driver->nactivevms++; + } else { + vm->def->id = -1; + close(vm->monitor); + vm->monitor = -1; + } + + vm = vm->next; + } + return 0; } @@ -942,6 +946,7 @@ { VIR_FREE(driver->configDir); VIR_FREE(driver->stateDir); + VIR_FREE(driver->logDir); VIR_FREE(driver); } @@ -976,37 +981,6 @@ /* Otherwise we're happy to deal with a shutdown */ return 0; -} - -/** - * lxcSigHandler: - * @siginfo: Pointer to siginfo_t structure - * - * Handles signals received by libvirtd. Currently this is used to - * catch SIGCHLD from an exiting container. - * - * Returns 0 on success or -1 in case of error - */ -static int lxcSigHandler(siginfo_t *siginfo) -{ - int rc = -1; - lxc_vm_t *vm; - - if (siginfo->si_signo == SIGCHLD) { - vm = lxcFindVMByID(lxc_driver, siginfo->si_pid); - - if (NULL == vm) { - DEBUG("Ignoring SIGCHLD from non-container process %d\n", - siginfo->si_pid); - goto cleanup; - } - - rc = lxcVMCleanup(lxc_driver, vm); - - } - -cleanup: - return rc; } @@ -1079,7 +1053,7 @@ lxcShutdown, NULL, /* reload */ lxcActive, - lxcSigHandler + NULL, }; int lxcRegister(void) diff -r 8093fb566748 src/util.c --- a/src/util.c Fri Aug 01 14:47:33 2008 +0100 +++ b/src/util.c Tue Aug 05 12:13:24 2008 +0100 @@ -37,6 +37,7 @@ #include <sys/wait.h> #endif #include <string.h> +#include <termios.h> #include "c-ctype.h" #ifdef HAVE_PATHS_H @@ -556,6 +557,163 @@ return 0; } + +int virFileOpenTty(int *ttymaster, + char **ttyName, + int rawmode) +{ + int rc = -1; + + if ((*ttymaster = posix_openpt(O_RDWR|O_NOCTTY|O_NONBLOCK)) < 0) + goto cleanup; + + if (unlockpt(*ttymaster) < 0) + goto cleanup; + + if (grantpt(*ttymaster) < 0) + goto cleanup; + + if (rawmode) { + struct termios ttyAttr; + if (tcgetattr(*ttymaster, &ttyAttr) < 0) + goto cleanup; + + cfmakeraw(&ttyAttr); + + if (tcsetattr(*ttymaster, TCSADRAIN, &ttyAttr) < 0) + goto cleanup; + } + + if (ttyName) { + char tempTtyName[PATH_MAX]; + if (ptsname_r(*ttymaster, tempTtyName, sizeof(tempTtyName)) < 0) + goto cleanup; + + if ((*ttyName = strdup(tempTtyName)) == NULL) { + errno = ENOMEM; + goto cleanup; + } + } + + rc = 0; + +cleanup: + if (rc != 0 && + *ttymaster != -1) { + close(*ttymaster); + } + + return rc; + +} + + +int virFileWritePid(const char *dir, + const char *name, + pid_t pid) +{ + int rc; + int fd; + FILE *file = NULL; + char *pidfile = NULL; + + if ((rc = virFileMakePath(dir))) { + goto cleanup; + } + + if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) { + rc = ENOMEM; + goto cleanup; + } + + if ((fd = open(pidfile, + O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR)) < 0) { + rc = errno; + goto cleanup; + } + + if (!(file = fdopen(fd, "w"))) { + rc = errno; + close(fd); + goto cleanup; + } + + if (fprintf(file, "%d", pid) < 0) { + rc = errno; + goto cleanup; + } + + rc = 0; + +cleanup: + if (file && + fclose(file) < 0) { + rc = errno; + } + + VIR_FREE(pidfile); + return rc; +} + +int virFileReadPid(const char *dir, + const char *name, + pid_t *pid) +{ + int rc; + FILE *file; + char *pidfile = NULL; + *pid = 0; + if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) { + rc = ENOMEM; + goto cleanup; + } + + if (!(file = fopen(pidfile, "r"))) { + rc = errno; + goto cleanup; + } + + if (fscanf(file, "%d", pid) != 1) { + rc = EINVAL; + goto cleanup; + } + + if (fclose(file) < 0) { + rc = errno; + goto cleanup; + } + + rc = 0; + + cleanup: + VIR_FREE(pidfile); + return rc; +} + +int virFileDeletePid(const char *dir, + const char *name) +{ + int rc = 0; + char *pidfile = NULL; + + if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) { + rc = errno; + goto cleanup; + } + + if (unlink(pidfile) < 0 && + errno != ENOENT) { + rc = errno; + } + +cleanup: + VIR_FREE(pidfile); + return rc; +} + + + /* Like strtol, but produce an "int" result, and check more carefully. Return 0 upon success; return -1 to indicate failure. When END_PTR is NULL, the byte after the final valid digit must be NUL. diff -r 8093fb566748 src/util.h --- a/src/util.h Fri Aug 01 14:47:33 2008 +0100 +++ b/src/util.h Tue Aug 05 12:13:24 2008 +0100 @@ -58,6 +58,19 @@ char *buf, unsigned int buflen); +int virFileOpenTty(int *ttymaster, + char **ttyName, + int rawmode); + + +int virFileWritePid(const char *dir, + const char *name, + pid_t pid); +int virFileReadPid(const char *dir, + const char *name, + pid_t *pid); +int virFileDeletePid(const char *dir, + const char *name); int __virStrToLong_i(char const *s, char **end_ptr, -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|

This patch removes the configDir and autostartLink paths from the internal virDomainObjPtr struct. Instead they can calculated at time of use. This is to facilitate a following patch, where we want to write a config to an alternative location at some points. This also makes the LXC & QEMU drivers correctly set/use the 'persistent' property. domain_conf.c | 99 +++++++++++++++++++++++++++++----------------------------- domain_conf.h | 8 +--- lxc_driver.c | 40 ++++++++++++++++++----- qemu_driver.c | 71 ++++++++++++++++++++++++++++++----------- 4 files changed, 138 insertions(+), 80 deletions(-) Daniel diff -r a204a9425afd src/domain_conf.c --- a/src/domain_conf.c Tue Aug 05 16:50:49 2008 +0100 +++ b/src/domain_conf.c Tue Aug 05 16:50:51 2008 +0100 @@ -399,8 +399,6 @@ virDomainDefFree(dom->newDef); VIR_FREE(dom->vcpupids); - VIR_FREE(dom->configFile); - VIR_FREE(dom->autostartLink); VIR_FREE(dom); } @@ -2952,31 +2950,23 @@ int virDomainSaveConfig(virConnectPtr conn, const char *configDir, - const char *autostartDir, - virDomainObjPtr dom) + virDomainDefPtr def) { char *xml; + char *configFile = NULL; int fd = -1, ret = -1; size_t towrite; int err; - if (!dom->configFile && - asprintf(&dom->configFile, "%s/%s.xml", - configDir, dom->def->name) < 0) { - dom->configFile = NULL; - virDomainReportError(conn, VIR_ERR_NO_MEMORY, NULL); - goto cleanup; - } - if (!dom->autostartLink && - asprintf(&dom->autostartLink, "%s/%s.xml", - autostartDir, dom->def->name) < 0) { - dom->autostartLink = NULL; + if (asprintf(&configFile, "%s/%s.xml", + configDir, def->name) < 0) { + configFile = NULL; virDomainReportError(conn, VIR_ERR_NO_MEMORY, NULL); goto cleanup; } if (!(xml = virDomainDefFormat(conn, - dom->newDef ? dom->newDef : dom->def, + def, VIR_DOMAIN_XML_SECURE))) goto cleanup; @@ -2987,34 +2977,27 @@ goto cleanup; } - if ((err = virFileMakePath(autostartDir))) { - virDomainReportError(conn, VIR_ERR_INTERNAL_ERROR, - _("cannot create autostart directory %s: %s"), - autostartDir, strerror(err)); - goto cleanup; - } - - if ((fd = open(dom->configFile, + if ((fd = open(configFile, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR )) < 0) { virDomainReportError(conn, VIR_ERR_INTERNAL_ERROR, - _("cannot create config file %s: %s"), - dom->configFile, strerror(errno)); + _("cannot create config file %s: %s"), + configFile, strerror(errno)); goto cleanup; } towrite = strlen(xml); if (safewrite(fd, xml, towrite) < 0) { virDomainReportError(conn, VIR_ERR_INTERNAL_ERROR, - _("cannot write config file %s: %s"), - dom->configFile, strerror(errno)); + _("cannot write config file %s: %s"), + configFile, strerror(errno)); goto cleanup; } if (close(fd) < 0) { virDomainReportError(conn, VIR_ERR_INTERNAL_ERROR, - _("cannot save config file %s: %s"), - dom->configFile, strerror(errno)); + _("cannot save config file %s: %s"), + configFile, strerror(errno)); goto cleanup; } @@ -3072,8 +3055,6 @@ goto error; dom->state = VIR_DOMAIN_SHUTOFF; - dom->configFile = configFile; - dom->autostartLink = autostartLink; dom->autostart = autostart; return dom; @@ -3104,6 +3085,8 @@ } while ((entry = readdir(dir))) { + virDomainObjPtr dom; + if (entry->d_name[0] == '.') continue; @@ -3112,12 +3095,14 @@ /* NB: ignoring errors, so one malformed config doesn't kill the whole process */ - virDomainLoadConfig(conn, - caps, - doms, - configDir, - autostartDir, - entry->d_name); + dom = virDomainLoadConfig(conn, + caps, + doms, + configDir, + autostartDir, + entry->d_name); + if (dom) + dom->persistent = 1; } closedir(dir); @@ -3126,25 +3111,43 @@ } int virDomainDeleteConfig(virConnectPtr conn, - virDomainObjPtr dom) + const char *configDir, + const char *autostartDir, + virDomainObjPtr dom) { - if (!dom->configFile || !dom->autostartLink) { - virDomainReportError(conn, VIR_ERR_INTERNAL_ERROR, - _("no config file for %s"), dom->def->name); - return -1; + char *configFile = NULL, *autostartLink = NULL; + int ret = -1; + + if (asprintf(&configFile, "%s/%s", + configDir, dom->def->name) < 0) { + configFile = NULL; + virDomainReportError(conn, VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + if (asprintf(&autostartLink, "%s/%s", + autostartDir, dom->def->name) < 0) { + autostartLink = NULL; + virDomainReportError(conn, VIR_ERR_NO_MEMORY, NULL); + goto cleanup; } /* Not fatal if this doesn't work */ - unlink(dom->autostartLink); + unlink(autostartLink); - if (unlink(dom->configFile) < 0) { + if (unlink(configFile) < 0 && + errno != ENOENT) { virDomainReportError(conn, VIR_ERR_INTERNAL_ERROR, - _("cannot remove config for %s: %s"), + _("cannot remove config for %s: %s"), dom->def->name, strerror(errno)); - return -1; + goto cleanup; } - return 0; + ret = 0; + +cleanup: + VIR_FREE(configFile); + VIR_FREE(autostartLink); + return ret; } #endif /* ! PROXY */ diff -r a204a9425afd src/domain_conf.h --- a/src/domain_conf.h Tue Aug 05 16:50:49 2008 +0100 +++ b/src/domain_conf.h Tue Aug 05 16:50:51 2008 +0100 @@ -409,9 +409,6 @@ unsigned int autostart : 1; unsigned int persistent : 1; - char *configFile; - char *autostartLink; - virDomainDefPtr def; /* The current definition */ virDomainDefPtr newDef; /* New definition to activate at shutdown */ @@ -482,8 +479,7 @@ int virDomainSaveConfig(virConnectPtr conn, const char *configDir, - const char *autostartDir, - virDomainObjPtr dom); + virDomainDefPtr def); virDomainObjPtr virDomainLoadConfig(virConnectPtr conn, virCapsPtr caps, @@ -499,6 +495,8 @@ const char *autostartDir); int virDomainDeleteConfig(virConnectPtr conn, + const char *configDir, + const char *autostartDir, virDomainObjPtr dom); virDomainNetDefPtr virDomainNetDefParseXML(virConnectPtr conn, diff -r a204a9425afd src/lxc_driver.c --- a/src/lxc_driver.c Tue Aug 05 16:50:49 2008 +0100 +++ b/src/lxc_driver.c Tue Aug 05 16:50:51 2008 +0100 @@ -250,11 +250,11 @@ virDomainDefFree(def); return NULL; } + vm->persistent = 1; if (virDomainSaveConfig(conn, driver->configDir, - driver->autostartDir, - vm) < 0) { + vm->newDef ? vm->newDef : vm->def) < 0) { virDomainRemoveInactive(&driver->domains, vm); return NULL; } @@ -284,10 +284,17 @@ return -1; } - if (virDomainDeleteConfig(dom->conn, vm) <0) + if (!vm->persistent) { + lxcError(dom->conn, dom, VIR_ERR_INTERNAL_ERROR, + "%s", _("cannot undefine transient domain")); return -1; + } - vm->configFile[0] = '\0'; + if (virDomainDeleteConfig(dom->conn, + driver->configDir, + driver->autostartDir, + vm) <0) + return -1; virDomainRemoveInactive(&driver->domains, vm); @@ -639,8 +646,13 @@ return; } - if (lxcVmTerminate(NULL, vm, SIGINT) < 0) - virEventRemoveHandle(fd); + lxcVmTerminate(NULL, vm, SIGINT); + + virEventRemoveHandle(fd); + + if (!vm->persistent) + virDomainRemoveInactive(&driver->domains, + vm); } @@ -846,6 +858,7 @@ { lxc_driver_t *driver = (lxc_driver_t*)dom->conn->privateData; virDomainObjPtr vm = virDomainFindByID(driver->domains, dom->id); + int ret; if (!vm) { lxcError(dom->conn, dom, VIR_ERR_INVALID_DOMAIN, @@ -853,7 +866,12 @@ return -1; } - return lxcVmTerminate(dom->conn, vm, 0); + ret = lxcVmTerminate(dom->conn, vm, 0); + if (!vm->persistent) + virDomainRemoveInactive(&driver->domains, + vm); + + return ret; } @@ -869,6 +887,7 @@ { lxc_driver_t *driver = (lxc_driver_t*)dom->conn->privateData; virDomainObjPtr vm = virDomainFindByID(driver->domains, dom->id); + int ret; if (!vm) { lxcError(dom->conn, dom, VIR_ERR_INVALID_DOMAIN, @@ -876,7 +895,12 @@ return -1; } - return lxcVmTerminate(dom->conn, vm, SIGKILL); + ret = lxcVmTerminate(dom->conn, vm, SIGKILL); + if (!vm->persistent) + virDomainRemoveInactive(&driver->domains, + vm); + + return ret; } static int lxcCheckNetNsSupport(void) diff -r a204a9425afd src/qemu_driver.c --- a/src/qemu_driver.c Tue Aug 05 16:50:49 2008 +0100 +++ b/src/qemu_driver.c Tue Aug 05 16:50:51 2008 +0100 @@ -351,7 +351,7 @@ virDomainObjPtr next = vm->next; if (virDomainIsActive(vm)) qemudShutdownVMDaemon(NULL, qemu_driver, vm); - if (!vm->configFile) + if (!vm->persistent) virDomainRemoveInactive(&qemu_driver->domains, vm); vm = next; @@ -1068,7 +1068,7 @@ static int qemudDispatchVMLog(struct qemud_driver *driver, virDomainObjPtr vm, int fd) { if (qemudVMData(driver, vm, fd) < 0) { qemudShutdownVMDaemon(NULL, driver, vm); - if (!vm->configFile) + if (!vm->persistent) virDomainRemoveInactive(&driver->domains, vm); } @@ -1078,7 +1078,7 @@ static int qemudDispatchVMFailure(struct qemud_driver *driver, virDomainObjPtr vm, int fd ATTRIBUTE_UNUSED) { qemudShutdownVMDaemon(NULL, driver, vm); - if (!vm->configFile) + if (!vm->persistent) virDomainRemoveInactive(&driver->domains, vm); return 0; @@ -2138,7 +2138,7 @@ } qemudShutdownVMDaemon(dom->conn, driver, vm); - if (!vm->configFile) + if (!vm->persistent) virDomainRemoveInactive(&driver->domains, vm); @@ -2469,7 +2469,7 @@ /* Shut it down */ qemudShutdownVMDaemon(dom->conn, driver, vm); - if (!vm->configFile) + if (!vm->persistent) virDomainRemoveInactive(&driver->domains, vm); @@ -2761,7 +2761,7 @@ if (ret < 0) { qemudReportError(conn, NULL, NULL, VIR_ERR_OPERATION_FAILED, "%s", _("failed to start VM")); - if (!vm->configFile) + if (!vm->persistent) virDomainRemoveInactive(&driver->domains, vm); return -1; @@ -2867,11 +2867,11 @@ virDomainDefFree(def); return NULL; } + vm->persistent = 1; if (virDomainSaveConfig(conn, driver->configDir, - driver->autostartDir, - vm) < 0) { + vm->newDef ? vm->newDef : vm->def) < 0) { virDomainRemoveInactive(&driver->domains, vm); return NULL; @@ -2898,7 +2898,13 @@ return -1; } - if (virDomainDeleteConfig(dom->conn, vm) < 0) + if (!vm->persistent) { + qemudReportError(dom->conn, dom, NULL, VIR_ERR_INTERNAL_ERROR, + "%s", _("cannot undefine transient domain")); + return -1; + } + + if (virDomainDeleteConfig(dom->conn, driver->configDir, driver->autostartDir, vm) < 0) return -1; virDomainRemoveInactive(&driver->domains, @@ -3024,13 +3030,21 @@ } static int qemudDomainSetAutostart(virDomainPtr dom, - int autostart) { + int autostart) { struct qemud_driver *driver = (struct qemud_driver *)dom->conn->privateData; virDomainObjPtr vm = virDomainFindByUUID(driver->domains, dom->uuid); + char *configFile = NULL, *autostartLink = NULL; + int ret = -1; if (!vm) { qemudReportError(dom->conn, dom, NULL, VIR_ERR_INVALID_DOMAIN, "%s", _("no domain with matching uuid")); + return -1; + } + + if (!vm->persistent) { + qemudReportError(dom->conn, dom, NULL, VIR_ERR_INTERNAL_ERROR, + "%s", _("cannot set autostart for transient domain")); return -1; } @@ -3039,6 +3053,20 @@ if (vm->autostart == autostart) return 0; + if (asprintf(&configFile, "%s/%s.xml", + driver->configDir, vm->def->name) < 0) { + configFile = NULL; + qemudReportError(dom->conn, dom, NULL, VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + + if (asprintf(&autostartLink, "%s/%s.xml", + driver->autostartDir, vm->def->name) < 0) { + autostartLink = NULL; + qemudReportError(dom->conn, dom, NULL, VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + if (autostart) { int err; @@ -3046,27 +3074,32 @@ qemudReportError(dom->conn, dom, NULL, VIR_ERR_INTERNAL_ERROR, _("cannot create autostart directory %s: %s"), driver->autostartDir, strerror(err)); - return -1; + goto cleanup; } - if (symlink(vm->configFile, vm->autostartLink) < 0) { + if (symlink(configFile, autostartLink) < 0) { qemudReportError(dom->conn, dom, NULL, VIR_ERR_INTERNAL_ERROR, - _("Failed to create symlink '%s' to '%s': %s"), - vm->autostartLink, vm->configFile, strerror(errno)); - return -1; + _("Failed to create symlink '%s to '%s': %s"), + autostartLink, configFile, strerror(errno)); + goto cleanup; } } else { - if (unlink(vm->autostartLink) < 0 && errno != ENOENT && errno != ENOTDIR) { + if (unlink(autostartLink) < 0 && errno != ENOENT && errno != ENOTDIR) { qemudReportError(dom->conn, dom, NULL, VIR_ERR_INTERNAL_ERROR, _("Failed to delete symlink '%s': %s"), - vm->autostartLink, strerror(errno)); - return -1; + autostartLink, strerror(errno)); + goto cleanup; } } vm->autostart = autostart; + ret = 0; - return 0; +cleanup: + VIR_FREE(configFile); + VIR_FREE(autostartLink); + + return ret; } /* This uses the 'info blockstats' monitor command which was -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|

Internally the drivers track the current live configuration, and the new inactive config for running domains. When the libvirtd process is restarted though, this data is lost for any active LXC domains. This patch makes the LXC driver persist the live config to /var/run/libvirt/lxc/NAME.xml so it can be tracked across restarts It required a small change to the domain XML APis to make the autostart symlink processing optional when deleting a config file domain_conf.c | 24 ++++++++++++------------ lxc_driver.c | 27 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) Daniel diff -r cf1cf3a1d4d6 src/domain_conf.c --- a/src/domain_conf.c Tue Aug 05 16:50:51 2008 +0100 +++ b/src/domain_conf.c Tue Aug 05 16:50:59 2008 +0100 @@ -1087,13 +1087,11 @@ break; case VIR_DOMAIN_CHR_TYPE_PTY: - /* @path attribute is an output only property - pty is auto-allocted */ - break; - case VIR_DOMAIN_CHR_TYPE_DEV: case VIR_DOMAIN_CHR_TYPE_FILE: case VIR_DOMAIN_CHR_TYPE_PIPE: - if (path == NULL) { + if (path == NULL && + def->type != VIR_DOMAIN_CHR_TYPE_PTY) { virDomainReportError(conn, VIR_ERR_INTERNAL_ERROR, "%s", _("Missing source path attribute for char device")); goto error; @@ -3124,15 +3122,17 @@ virDomainReportError(conn, VIR_ERR_NO_MEMORY, NULL); goto cleanup; } - if (asprintf(&autostartLink, "%s/%s", - autostartDir, dom->def->name) < 0) { - autostartLink = NULL; - virDomainReportError(conn, VIR_ERR_NO_MEMORY, NULL); - goto cleanup; + if (autostartDir) { + if (asprintf(&autostartLink, "%s/%s", + autostartDir, dom->def->name) < 0) { + autostartLink = NULL; + virDomainReportError(conn, VIR_ERR_NO_MEMORY, NULL); + goto cleanup; + } + + /* Not fatal if this doesn't work */ + unlink(autostartLink); } - - /* Not fatal if this doesn't work */ - unlink(autostartLink); if (unlink(configFile) < 0 && errno != ENOENT) { diff -r cf1cf3a1d4d6 src/lxc_driver.c --- a/src/lxc_driver.c Tue Aug 05 16:50:51 2008 +0100 +++ b/src/lxc_driver.c Tue Aug 05 16:50:59 2008 +0100 @@ -399,6 +399,7 @@ close(vm->monitor); virFileDeletePid(driver->stateDir, vm->def->name); + virDomainDeleteConfig(conn, driver->stateDir, NULL, vm); vm->state = VIR_DOMAIN_SHUTOFF; vm->pid = -1; @@ -615,6 +616,12 @@ if (signum == 0) signum = SIGINT; + if (vm->pid <= 0) { + lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, + _("invalid PID %d for container"), vm->pid); + return -1; + } + if (kill(vm->pid, signum) < 0) { if (errno != ESRCH) { lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, @@ -741,6 +748,12 @@ lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR, _("Failed to read pid file %s/%s.pid: %s"), driver->stateDir, vm->def->name, strerror(errno)); + rc = -1; + goto cleanup; + } + + /* Persist the live configuration */ + if (virDomainSaveConfig(conn, driver->stateDir, vm->def) < 0) { rc = -1; goto cleanup; } @@ -960,6 +973,8 @@ vm = lxc_driver->domains; while (vm) { + char *config = NULL; + virDomainDefPtr tmp; int rc; if ((vm->monitor = lxcMonitorClient(NULL, lxc_driver, vm)) < 0) { vm = vm->next; @@ -972,6 +987,18 @@ vm->monitor = -1; vm = vm->next; continue; + } + + if (asprintf(&config, "%s/%s.xml", + lxc_driver->stateDir, vm->def->name) < 0) + continue; + + /* Try and load the live config */ + tmp = virDomainDefParseFile(NULL, lxc_driver->caps, config); + VIR_FREE(config); + if (tmp) { + vm->newDef = vm->def; + vm->def = tmp; } if (vm->pid != 0) { -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|

The LXC driver currently allows custom mount points to be setup inside the container. This only works for non-root mount points. You cannot replace the entire root filesystem. This patch adds support for replacing the entire root filesystem, thus allowing the use of LXC containers as a 'better chroot than chroot'. Well, with one minor flaw - the Linux kernel currently has no device namespace virtualization, so the admin inside the container can just do a 'mknod' and access the real devices of the host. So for now this patch doesn't make LXC containers secure, but a future kernel release will enable it to be secure. lxc_container.c | 253 ++++++++++++++++++++++++++++++++++++++++++++++++-------- util.c | 12 +- 2 files changed, 226 insertions(+), 39 deletions(-) Daniel diff -r eaa42985aed4 src/lxc_container.c --- a/src/lxc_container.c Tue Aug 05 16:50:59 2008 +0100 +++ b/src/lxc_container.c Tue Aug 05 16:51:14 2008 +0100 @@ -1,10 +1,12 @@ /* * Copyright IBM Corp. 2008 + * Copyright Red Hat 2008 * * lxc_container.c: file description * * Authors: * David L. Leskovec <dlesko at linux.vnet.ibm.com> + * Daniel P. Berrange <berrange@redhat.com> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -28,10 +30,18 @@ #include <fcntl.h> #include <limits.h> #include <stdlib.h> +#include <stdio.h> #include <sys/ioctl.h> #include <sys/mount.h> #include <sys/wait.h> #include <unistd.h> +#include <mntent.h> + +/* Yes, we want linux private one, for _syscall2() macro */ +#include <linux/unistd.h> + +/* For MS_MOVE */ +#include <linux/fs.h> #include "lxc_container.h" #include "util.h" @@ -105,23 +115,15 @@ * * Returns 0 on success or -1 in case of error */ -static int lxcContainerSetStdio(int control, const char *ttyPath) +static int lxcContainerSetStdio(int control, int ttyfd) { int rc = -1; - int ttyfd; int open_max, i; if (setsid() < 0) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, _("setsid failed: %s"), strerror(errno)); - goto error_out; - } - - ttyfd = open(ttyPath, O_RDWR|O_NOCTTY); - if (ttyfd < 0) { - lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("open(%s) failed: %s"), ttyPath, strerror(errno)); - goto error_out; + goto cleanup; } if (ioctl(ttyfd, TIOCSCTTY, NULL) < 0) { @@ -159,8 +161,6 @@ cleanup: close(ttyfd); - -error_out: return rc; } @@ -223,6 +223,7 @@ return 0; } + /** * lxcEnableInterfaces: * @vm: Pointer to vm structure @@ -252,6 +253,20 @@ return rc; } + +//_syscall2(int, pivot_root, char *, newroot, const char *, oldroot) +extern int pivot_root(const char * new_root,const char * put_old); + +static int lxcContainerChildMountSort(const void *a, const void *b) +{ + const char **sa = (const char**)a; + const char **sb = (const char**)b; + + /* Delibrately reversed args - we need to unmount deepest + children first */ + return strcmp(*sb, *sa); +} + /** * lxcChild: * @argv: Pointer to container arguments @@ -269,8 +284,8 @@ int rc = -1; lxc_child_argv_t *argv = data; virDomainDefPtr vmDef = argv->config; - virDomainFSDefPtr curMount; - int i; + virDomainFSDefPtr tmp, root = NULL; + int ttyfd, i; if (NULL == vmDef) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, @@ -278,36 +293,210 @@ return -1; } +#if 0 + ttyfd = open(argv->ttyPath, O_RDWR|O_NOCTTY); + if (ttyfd < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("open(%s) failed: %s"), argv->ttyPath, strerror(errno)); + return -1; + } +#endif + /* handle the bind mounts first before doing anything else that may */ /* then access those mounted dirs */ - curMount = vmDef->fss; - for (i = 0; curMount; curMount = curMount->next) { - // XXX fix - if (curMount->type != VIR_DOMAIN_FS_TYPE_MOUNT) + for (tmp = vmDef->fss; tmp && !root; tmp = tmp->next) { + if (tmp->type != VIR_DOMAIN_FS_TYPE_MOUNT) continue; - rc = mount(curMount->src, - curMount->dst, - NULL, - MS_BIND, - NULL); - if (0 != rc) { + if (STREQ(tmp->dst, "/")) + root = tmp; + } + + if (root) { + char *oldroot; + struct mntent *mntent; + char **mounts = NULL; + int nmounts = 0; + FILE *procmnt; + struct { + int maj; + int min; + const char *path; + } devs[] = { + { 1, 3, "/dev/null" }, + { 1, 5, "/dev/zero" }, + { 1, 7, "/dev/full" }, + { 5, 1, "/dev/console" }, + }; + + /* Got a FS mapped to /, we're going the pivot_root + approach to do a better-chroot-than-chroot */ + + /* this is based on this thread http://lkml.org/lkml/2008/3/5/29 */ + + /* First step is to ensure the new root itself is + a mount point */ + if (mount(root->src, root->src, NULL, MS_BIND, NULL) < 0) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("failed to mount %s at %s for container: %s"), - curMount->src, curMount->dst, strerror(errno)); + _("failed to bind new root %s: %s"), + root->src, strerror(errno)); + return -1; + } + + if (asprintf(&oldroot, "%s/.oldroot", root->src) < 0) { + lxcError(NULL, NULL, VIR_ERR_NO_MEMORY, NULL); + return -1; + } + + if (virFileMakePath(oldroot) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to create %s: %s"), + oldroot, strerror(errno)); + return -1; + } + + /* The old root directory will live at /.oldroot after + * this and will soon be unmounted completely */ + if (pivot_root(root->src, oldroot) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to pivot root %s to %s: %s"), + oldroot, root->src, strerror(errno)); + return -1; + } + + /* CWD is undefined after pivot_root, so go to / */ + if (chdir("/") < 0) { + return -1; + } + + if (virFileMakePath("/proc") < 0 || + mount("none", "/proc", "proc", 0, NULL) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to mount /proc for container: %s"), + strerror(errno)); + return -1; + } + if (virFileMakePath("/dev") < 0 || + mount("none", "/dev", "tmpfs", 0, NULL) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to mount /dev tmpfs for container: %s"), + strerror(errno)); + return -1; + } + /* Move old devpts into container, since we have to + connect to the master ptmx which was opened in + the parent. + XXX This sucks, we need to figure out how to get our + own private devpts for isolation + */ + if (virFileMakePath("/dev/pts") < 0 || + mount("/.oldroot/dev/pts", "/dev/pts", NULL, + MS_MOVE, NULL) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to move /dev/pts into container: %s"), + strerror(errno)); + return -1; + } + + /* Populate /dev/ with a few important bits */ + umask(0); + for (i = 0 ; i < ARRAY_CARDINALITY(devs) ; i++) { + dev_t dev = makedev(devs[i].maj, devs[i].min); + if (mknod(devs[i].path, + 0777 | S_IFCHR, + dev) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to make device %s: %s"), + devs[i].path, strerror(errno)); + return -1; + } + } + umask(0700); + + /* Pull in rest of container's mounts */ + for (tmp = vmDef->fss; tmp; tmp = tmp->next) { + char *src; + if (STREQ(tmp->dst, "/")) + continue; + // XXX fix + if (tmp->type != VIR_DOMAIN_FS_TYPE_MOUNT) + continue; + + if (asprintf(&src, "/.oldroot/%s", tmp->src) < 0) + return -1; + + if (virFileMakePath(tmp->dst) < 0 || + mount(src, tmp->dst, NULL, MS_BIND, NULL) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to mount %s at %s for container: %s"), + tmp->src, tmp->dst, strerror(errno)); + return -1; + } + VIR_FREE(src); + } + + if (!(procmnt = setmntent("/proc/mounts", "r"))) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to read /proc/mounts: %s"), + strerror(errno)); + return -1; + } + while ((mntent = getmntent(procmnt)) != NULL) { + if (!STRPREFIX(mntent->mnt_dir, "/.oldroot")) + continue; + if (VIR_REALLOC_N(mounts, nmounts+1) < 0) + return -1; + mounts[nmounts++] = strdup(mntent->mnt_dir); + } + endmntent(procmnt); + + qsort(mounts, nmounts, sizeof(mounts[0]), + lxcContainerChildMountSort); + + for (i = 0 ; i < nmounts ; i++) { + if (umount(mounts[i]) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to unmount %s: %s"), + mounts[i], strerror(errno)); + return -1; + } + } + } else { + /* Nothing mapped to /, we're using the main root, + but with extra stuff mapped in */ + for (tmp = vmDef->fss; tmp; tmp = tmp->next) { + // XXX fix + if (tmp->type != VIR_DOMAIN_FS_TYPE_MOUNT) + continue; + rc = mount(tmp->src, + tmp->dst, + NULL, + MS_BIND, + NULL); + if (0 != rc) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to mount %s at %s for container: %s"), + tmp->src, tmp->dst, strerror(errno)); + return -1; + } + } + + /* mount /proc */ + if (mount("lxcproc", "/proc", "proc", 0, NULL) < 0) { + lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, + _("failed to mount /proc for container: %s"), + strerror(errno)); return -1; } } - /* mount /proc */ - rc = mount("lxcproc", "/proc", "proc", 0, NULL); - if (0 != rc) { + ttyfd = open(argv->ttyPath, O_RDWR|O_NOCTTY); + if (ttyfd < 0) { lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR, - _("failed to mount /proc for container: %s"), - strerror(errno)); + _("open(%s) failed: %s"), argv->ttyPath, strerror(errno)); return -1; } - if (lxcContainerSetStdio(argv->monitor, argv->ttyPath) < 0) + if (lxcContainerSetStdio(argv->monitor, ttyfd) < 0) return -1; /* Wait for interface devices to show up */ diff -r eaa42985aed4 src/util.c --- a/src/util.c Tue Aug 05 16:50:59 2008 +0100 +++ b/src/util.c Tue Aug 05 16:51:14 2008 +0100 @@ -524,13 +524,11 @@ if (!(p = strrchr(parent, '/'))) return EINVAL; - if (p == parent) - return EPERM; - - *p = '\0'; - - if ((err = virFileMakePath(parent))) - return err; + if (p != parent) { + *p = '\0'; + if ((err = virFileMakePath(parent))) + return err; + } if (mkdir(path, 0777) < 0 && errno != EEXIST) return errno; -- |: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|