8:22 a.m.
From: "Daniel P. Berrange" <berrange(a)redhat.com>
This defines a new RPC protocol to be used between the LXC
controller and the libvirtd LXC driver. There is only a
single RPC message defined thus far, an asynchronous "EXIT"
event that is emitted just before the LXC controller process
exits. This provides the LXC driver with details about how
the container shutdown - normally, or abnormally (crashed),
thus allowing the driver to emit better libvirt events.
Emitting the event in the LXC controller requires a few
little tricks with the RPC service. Simply calling the
virNetServiceClientSendMessage does not work, since this
merely queues the message for asynchronous processing.
In addition the main event loop is no longer running at
the point the event is emitted, so no I/O is processed.
Thus after invoking virNetServiceClientSendMessage it is
necessary to mark the client as being in "delayed close"
mode. Then the event loop is run again, until the client
completes its close - this happens only after the queued
message has been fully transmitted. The final complexity
is that it is not safe to run virNetServerQuit() from the
client close callback, since that is invoked from a
context where the server is locked. Thus a zero-second
timer is used to trigger shutdown of the event loop,
causing the controller to finally exit.
* src/Makefile.am: Add rules for generating RPC protocol
files and dispatch methods
* src/lxc/lxc_controller.c: Emit an RPC event immediately
before exiting
* src/lxc/lxc_domain.h: Record the shutdown reason
given by the controller
* src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register
RPC program and event handler. Add callback to let
driver receive EXIT event.
* src/lxc/lxc_process.c: Use monitor exit event to decide
what kind of domain event to emit
* src/lxc/lxc_protocol.x: Define wire protocol for LXC
controller monitor.
Signed-off-by: Daniel P. Berrange <berrange(a)redhat.com>
---
.gitignore | 4 ++
src/Makefile.am | 50 +++++++++++++++++++
src/lxc/lxc_controller.c | 125 +++++++++++++++++++++++++++++++++++++++++++++-
src/lxc/lxc_domain.h | 1 +
src/lxc/lxc_monitor.c | 41 +++++++++++++++
src/lxc/lxc_monitor.h | 6 +++
src/lxc/lxc_process.c | 27 ++++++++++
src/lxc/lxc_protocol.x | 21 ++++++++
8 files changed, 274 insertions(+), 1 deletion(-)
create mode 100644 src/lxc/lxc_protocol.x
diff --git a/.gitignore b/.gitignore
index b4cbb5f..e4b3932 100644
--- a/.gitignore
+++ b/.gitignore
@@ -105,6 +105,10 @@
/src/locking/qemu-sanlock.conf
/src/locking/test_libvirt_sanlock.aug
/src/lxc/test_libvirtd_lxc.aug
+/src/lxc/lxc_controller_dispatch.h
+/src/lxc/lxc_monitor_dispatch.h
+/src/lxc/lxc_protocol.c
+/src/lxc/lxc_protocol.h
/src/qemu/test_libvirtd_qemu.aug
/src/remote/*_client_bodies.h
/src/remote/*_protocol.[ch]
diff --git a/src/Makefile.am b/src/Makefile.am
index 492ce73..44da1fa 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -347,7 +347,55 @@ if WITH_XEN_INOTIFY
XEN_DRIVER_SOURCES += xen/xen_inotify.c xen/xen_inotify.h
endif
+EXTRA_DIST += \
+ dtrace2systemtap.pl \
+ rpc/gendispatch.pl \
+ rpc/genprotocol.pl \
+ rpc/gensystemtap.pl \
+ rpc/virnetprotocol.x \
+ rpc/virkeepaliveprotocol.x
+
+LXC_PROTOCOL_GENERATED = \
+ $(srcdir)/lxc/lxc_protocol.h \
+ $(srcdir)/lxc/lxc_protocol.c \
+ $(NULL)
+
+LXC_MONITOR_GENERATED = \
+ $(srcdir)/lxc/lxc_monitor_dispatch.h \
+ $(NULL)
+
+LXC_CONTROLLER_GENERATED = \
+ $(srcdir)/lxc/lxc_controller_dispatch.h \
+ $(NULL)
+
+LXC_GENERATED = \
+ $(LXC_PROTOCOL_GENERATED) \
+ $(LXC_MONITOR_GENERATED) \
+ $(LXC_CONTROLLER_GENERATED) \
+ $(NULL)
+
+LXC_PROTOCOL = $(srcdir)/lxc/lxc_protocol.x
+
+$(srcdir)/lxc/lxc_monitor_dispatch.h: $(srcdir)/rpc/gendispatch.pl \
+ $(LXC_PROTOCOL)
+ $(AM_V_GEN)$(PERL) -w $(srcdir)/rpc/gendispatch.pl \
+ -k virLXCProtocol VIR_LXC_PROTOCOL $(LXC_PROTOCOL) > $@
+
+$(srcdir)/lxc/lxc_controller_dispatch.h: $(srcdir)/../src/rpc/gendispatch.pl \
+ $(REMOTE_PROTOCOL)
+ $(AM_V_GEN)$(PERL) -w $(srcdir)/rpc/gendispatch.pl \
+ -b virLXCProtocol VIR_LXC_PROTOCOL $(LXC_PROTOCOL) > $@
+
+EXTRA_DIST += \
+ $(LXC_PROTOCOL) \
+ $(LXC_GENERATED) \
+ $(NULL)
+
+BUILT_SOURCES += $(LXC_GENERATED)
+
LXC_DRIVER_SOURCES = \
+ $(LXC_PROTOCOL_GENERATED) \
+ $(LXC_MONITOR_GENERATED) \
lxc/lxc_conf.c lxc/lxc_conf.h \
lxc/lxc_container.c lxc/lxc_container.h \
lxc/lxc_cgroup.c lxc/lxc_cgroup.h \
@@ -357,6 +405,8 @@ LXC_DRIVER_SOURCES = \
lxc/lxc_driver.c lxc/lxc_driver.h
LXC_CONTROLLER_SOURCES = \
+ $(LXC_PROTOCOL_GENERATED) \
+ $(LXC_CONTROLLER_GENERATED) \
lxc/lxc_conf.c lxc/lxc_conf.h \
lxc/lxc_container.c lxc/lxc_container.h \
lxc/lxc_cgroup.c lxc/lxc_cgroup.h \
diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c
index e39c236..7fcc953 100644
--- a/src/lxc/lxc_controller.c
+++ b/src/lxc/lxc_controller.c
@@ -59,6 +59,7 @@
#include "lxc_conf.h"
#include "lxc_container.h"
#include "lxc_cgroup.h"
+#include "lxc_protocol.h"
#include "virnetdev.h"
#include "virnetdevveth.h"
#include "memory.h"
@@ -121,10 +122,25 @@ struct _virLXCController {
/* Server socket */
virNetServerPtr server;
+ virNetServerClientPtr client;
+ virNetServerProgramPtr prog;
+ bool inShutdown;
+ int timerShutdown;
};
+#include "lxc_controller_dispatch.h"
+
static void virLXCControllerFree(virLXCControllerPtr ctrl);
+static void virLXCControllerQuitTimer(int timer ATTRIBUTE_UNUSED, void *opaque)
+{
+ virLXCControllerPtr ctrl = opaque;
+
+ VIR_DEBUG("Triggering event loop quit");
+ virNetServerQuit(ctrl->server);
+}
+
+
static virLXCControllerPtr virLXCControllerNew(const char *name)
{
virLXCControllerPtr ctrl = NULL;
@@ -134,6 +150,8 @@ static virLXCControllerPtr virLXCControllerNew(const char *name)
if (VIR_ALLOC(ctrl) < 0)
goto no_memory;
+ ctrl->timerShutdown = -1;
+
if (!(ctrl->name = strdup(name)))
goto no_memory;
@@ -150,6 +168,11 @@ static virLXCControllerPtr virLXCControllerNew(const char *name)
0)) == NULL)
goto error;
+ if ((ctrl->timerShutdown = virEventAddTimeout(-1,
+ virLXCControllerQuitTimer, ctrl,
+ NULL)) < 0)
+ goto error;
+
cleanup:
VIR_FREE(configFile);
virCapabilitiesFree(caps);
@@ -238,6 +261,9 @@ static void virLXCControllerFree(virLXCControllerPtr ctrl)
virDomainDefFree(ctrl->def);
VIR_FREE(ctrl->name);
+ if (ctrl->timerShutdown != -1)
+ virEventRemoveTimeout(ctrl->timerShutdown);
+
virNetServerFree(ctrl->server);
VIR_FREE(ctrl);
@@ -539,12 +565,28 @@ static int virLXCControllerSetupResourceLimits(virLXCControllerPtr
ctrl)
}
+static void virLXCControllerClientCloseHook(virNetServerClientPtr client)
+{
+ virLXCControllerPtr ctrl = virNetServerClientGetPrivateData(client);
+
+ VIR_DEBUG("Client %p has closed", client);
+ if (ctrl->client == client)
+ ctrl->client = NULL;
+ if (ctrl->inShutdown) {
+ VIR_DEBUG("Arm timer to quit event loop");
+ virEventUpdateTimeout(ctrl->timerShutdown, 0);
+ }
+}
+
static int virLXCControllerClientHook(virNetServerPtr server ATTRIBUTE_UNUSED,
virNetServerClientPtr client,
void *opaque)
{
virLXCControllerPtr ctrl = opaque;
virNetServerClientSetPrivateData(client, ctrl, NULL);
+ virNetServerClientSetCloseHook(client, virLXCControllerClientCloseHook);
+ VIR_DEBUG("Got new client %p", client);
+ ctrl->client = client;
return 0;
}
@@ -581,6 +623,12 @@ static int virLXCControllerSetupServer(virLXCControllerPtr ctrl)
virNetServerServiceFree(svc);
svc = NULL;
+ if (!(ctrl->prog = virNetServerProgramNew(VIR_LXC_PROTOCOL_PROGRAM,
+ VIR_LXC_PROTOCOL_PROGRAM_VERSION,
+ virLXCProtocolProcs,
+ virLXCProtocolNProcs)))
+ goto error;
+
virNetServerUpdateServices(ctrl->server, true);
VIR_FREE(sockpath);
return 0;
@@ -1219,6 +1267,79 @@ virLXCControllerSetupConsoles(virLXCControllerPtr ctrl,
}
+static void
+virLXCControllerEventSend(virLXCControllerPtr ctrl,
+ int procnr,
+ xdrproc_t proc,
+ void *data)
+{
+ virNetMessagePtr msg;
+
+ if (!ctrl->client)
+ return;
+
+ VIR_DEBUG("Send event %d client=%p", procnr, ctrl->client);
+ if (!(msg = virNetMessageNew(false)))
+ goto cleanup;
+
+ msg->header.prog = virNetServerProgramGetID(ctrl->prog);
+ msg->header.vers = virNetServerProgramGetVersion(ctrl->prog);
+ msg->header.proc = procnr;
+ msg->header.type = VIR_NET_MESSAGE;
+ msg->header.serial = 1;
+ msg->header.status = VIR_NET_OK;
+
+ if (virNetMessageEncodeHeader(msg) < 0)
+ goto cleanup;
+
+ if (virNetMessageEncodePayload(msg, proc, data) < 0)
+ goto cleanup;
+
+ VIR_DEBUG("Queue event %d %zu", procnr, msg->bufferLength);
+ virNetServerClientSendMessage(ctrl->client, msg);
+
+ xdr_free(proc, data);
+ return;
+
+cleanup:
+ virNetMessageFree(msg);
+ xdr_free(proc, data);
+}
+
+
+static int
+virLXCControllerEventSendExit(virLXCControllerPtr ctrl,
+ int exitstatus)
+{
+ virLXCProtocolExitEventMsg msg;
+
+ VIR_DEBUG("Exit status %d", exitstatus);
+ memset(&msg, 0, sizeof(msg));
+ switch (exitstatus) {
+ case 0:
+ msg.status = VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN;
+ break;
+ default:
+ msg.status = VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR;
+ break;
+ }
+
+ virLXCControllerEventSend(ctrl,
+ VIR_LXC_PROTOCOL_PROC_EXIT_EVENT,
+ (xdrproc_t)xdr_virLXCProtocolExitEventMsg,
+ (void*)&msg);
+
+ if (ctrl->client) {
+ VIR_DEBUG("Waiting for client to complete dispatch");
+ ctrl->inShutdown = true;
+ virNetServerClientDelayedClose(ctrl->client);
+ virNetServerRun(ctrl->server);
+ }
+ VIR_DEBUG("Client has gone away");
+ return 0;
+}
+
+
static int
virLXCControllerRun(virLXCControllerPtr ctrl)
{
@@ -1306,6 +1427,8 @@ virLXCControllerRun(virLXCControllerPtr ctrl)
rc = virLXCControllerMain(ctrl);
+ virLXCControllerEventSendExit(ctrl, rc);
+
cleanup:
VIR_FORCE_CLOSE(control[0]);
VIR_FORCE_CLOSE(control[1]);
@@ -1527,5 +1650,5 @@ cleanup:
virLXCControllerFree(ctrl);
- return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+ return rc < 0? EXIT_FAILURE : EXIT_SUCCESS;
}
diff --git a/src/lxc/lxc_domain.h b/src/lxc/lxc_domain.h
index ff7c96f..9216c7a 100644
--- a/src/lxc/lxc_domain.h
+++ b/src/lxc/lxc_domain.h
@@ -31,6 +31,7 @@ typedef struct _virLXCDomainObjPrivate virLXCDomainObjPrivate;
typedef virLXCDomainObjPrivate *virLXCDomainObjPrivatePtr;
struct _virLXCDomainObjPrivate {
virLXCMonitorPtr monitor;
+ int shutoffReason;
};
void virLXCDomainSetPrivateDataHooks(virCapsPtr caps);
diff --git a/src/lxc/lxc_monitor.c b/src/lxc/lxc_monitor.c
index 6740aa7..e31b970 100644
--- a/src/lxc/lxc_monitor.c
+++ b/src/lxc/lxc_monitor.c
@@ -22,6 +22,8 @@
#include "lxc_monitor.h"
#include "lxc_conf.h"
+#include "lxc_protocol.h"
+#include "lxc_monitor_dispatch.h"
#include "memory.h"
@@ -41,9 +43,35 @@ struct _virLXCMonitor {
virLXCMonitorCallbacksPtr cb;
virNetClientPtr client;
+ virNetClientProgramPtr program;
};
static void virLXCMonitorFree(virLXCMonitorPtr mon);
+static void
+virLXCMonitorHandleEventExit(virNetClientProgramPtr prog,
+ virNetClientPtr client,
+ void *evdata, void *opaque);
+
+static virNetClientProgramEvent virLXCProtocolEvents[] = {
+ { VIR_LXC_PROTOCOL_PROC_EXIT_EVENT,
+ virLXCMonitorHandleEventExit,
+ sizeof(virLXCProtocolExitEventMsg),
+ (xdrproc_t)xdr_virLXCProtocolExitEventMsg },
+};
+
+
+static void
+virLXCMonitorHandleEventExit(virNetClientProgramPtr prog ATTRIBUTE_UNUSED,
+ virNetClientPtr client ATTRIBUTE_UNUSED,
+ void *evdata, void *opaque)
+{
+ virLXCMonitorPtr mon = opaque;
+ virLXCProtocolExitEventMsg *msg = evdata;
+
+ VIR_DEBUG("Event exit %d", msg->status);
+ if (mon->cb->exitNotify)
+ mon->cb->exitNotify(mon, msg->status, mon->vm);
+}
static void virLXCMonitorEOFNotify(virNetClientPtr client ATTRIBUTE_UNUSED,
@@ -54,6 +82,7 @@ static void virLXCMonitorEOFNotify(virNetClientPtr client
ATTRIBUTE_UNUSED,
virLXCMonitorCallbackEOFNotify eofNotify;
virDomainObjPtr vm;
+ VIR_DEBUG("EOF notify");
virLXCMonitorLock(mon);
eofNotify = mon->cb->eofNotify;
vm = mon->vm;
@@ -88,6 +117,17 @@ virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,
goto error;
+ if (!(mon->program = virNetClientProgramNew(VIR_LXC_PROTOCOL_PROGRAM,
+ VIR_LXC_PROTOCOL_PROGRAM_VERSION,
+ virLXCProtocolEvents,
+ ARRAY_CARDINALITY(virLXCProtocolEvents),
+ mon)))
+ goto error;
+
+ if (virNetClientAddProgram(mon->client,
+ mon->program) < 0)
+ goto error;
+
mon->vm = vm;
mon->cb = cb;
@@ -117,6 +157,7 @@ static void virLXCMonitorFree(virLXCMonitorPtr mon)
(mon->cb->destroy)(mon, mon->vm);
virMutexDestroy(&mon->lock);
virNetClientFree(mon->client);
+ virNetClientProgramFree(mon->program);
VIR_FREE(mon);
}
diff --git a/src/lxc/lxc_monitor.h b/src/lxc/lxc_monitor.h
index 32b8d36..cbcaffd 100644
--- a/src/lxc/lxc_monitor.h
+++ b/src/lxc/lxc_monitor.h
@@ -22,6 +22,7 @@
# define __LXC_MONITOR_H__
# include "domain_conf.h"
+# include "lxc_protocol.h"
typedef struct _virLXCMonitor virLXCMonitor;
typedef virLXCMonitor *virLXCMonitorPtr;
@@ -34,9 +35,14 @@ typedef void (*virLXCMonitorCallbackDestroy)(virLXCMonitorPtr mon,
typedef void (*virLXCMonitorCallbackEOFNotify)(virLXCMonitorPtr mon,
virDomainObjPtr vm);
+typedef void (*virLXCMonitorCallbackExitNotify)(virLXCMonitorPtr mon,
+ virLXCProtocolExitStatus status,
+ virDomainObjPtr vm);
+
struct _virLXCMonitorCallbacks {
virLXCMonitorCallbackDestroy destroy;
virLXCMonitorCallbackEOFNotify eofNotify;
+ virLXCMonitorCallbackExitNotify exitNotify;
};
virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,
diff --git a/src/lxc/lxc_process.c b/src/lxc/lxc_process.c
index 1bfd032..0b0a810 100644
--- a/src/lxc/lxc_process.c
+++ b/src/lxc/lxc_process.c
@@ -164,6 +164,9 @@ static void virLXCProcessCleanup(virLXCDriverPtr driver,
virLXCDomainObjPrivatePtr priv = vm->privateData;
virNetDevVPortProfilePtr vport = NULL;
+ VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d",
+ vm->def->name, (int)vm->pid, (int)reason);
+
/* now that we know it's stopped call the hook if present */
if (virHookPresent(VIR_HOOK_DRIVER_LXC)) {
char *xml = virDomainDefFormat(vm->def, 0);
@@ -517,10 +520,31 @@ static void virLXCProcessMonitorEOFNotify(virLXCMonitorPtr mon
ATTRIBUTE_UNUSED,
}
}
+static void virLXCProcessMonitorExitNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
+ virLXCProtocolExitStatus status,
+ virDomainObjPtr vm)
+{
+ virLXCDomainObjPrivatePtr priv = vm->privateData;
+
+ switch (status) {
+ case VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN:
+ priv->shutoffReason = VIR_DOMAIN_SHUTOFF_SHUTDOWN;
+ break;
+ case VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR:
+ priv->shutoffReason = VIR_DOMAIN_SHUTOFF_CRASHED;
+ break;
+ default:
+ priv->shutoffReason = VIR_DOMAIN_SHUTOFF_UNKNOWN;
+ break;
+ }
+ VIR_DEBUG("Domain shutoff reason %d (from status %d)",
+ priv->shutoffReason, status);
+}
static virLXCMonitorCallbacks monitorCallbacks = {
.eofNotify = virLXCProcessMonitorEOFNotify,
.destroy = virLXCProcessMonitorDestroy,
+ .exitNotify = virLXCProcessMonitorExitNotify,
};
@@ -555,6 +579,8 @@ int virLXCProcessStop(virLXCDriverPtr driver,
virCgroupPtr group = NULL;
int rc;
+ VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d",
+ vm->def->name, (int)vm->pid, (int)reason);
if (vm->pid <= 0) {
virReportError(VIR_ERR_INTERNAL_ERROR,
_("Invalid PID %d for container"), vm->pid);
@@ -988,6 +1014,7 @@ int virLXCProcessStart(virConnectPtr conn,
goto cleanup;
}
+ priv->shutoffReason = VIR_DOMAIN_SHUTOFF_UNKNOWN;
vm->def->id = vm->pid;
virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason);
diff --git a/src/lxc/lxc_protocol.x b/src/lxc/lxc_protocol.x
new file mode 100644
index 0000000..4fdbe34
--- /dev/null
+++ b/src/lxc/lxc_protocol.x
@@ -0,0 +1,21 @@
+/* -*- c -*-
+ * Define wire protocol for communication between the
+ * LXC driver in libvirtd, and the LXC controller in
+ * the libvirt_lxc helper program.
+ */
+
+enum virLXCProtocolExitStatus {
+ VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR,
+ VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN
+};
+
+struct virLXCProtocolExitEventMsg {
+ enum virLXCProtocolExitStatus status;
+};
+
+const VIR_LXC_PROTOCOL_PROGRAM = 0x12341234;
+const VIR_LXC_PROTOCOL_PROGRAM_VERSION = 1;
+
+enum virLXCProtocolProcedure {
+ VIR_LXC_PROTOCOL_PROC_EXIT_EVENT = 1 /* skipgen skipgen */
+};
--
1.7.10.4