[libvirt] [PATCH] lxc: suspend/resume support

Hi, This patch implements suspend/resume functions for LXC using cgroups freezer subsystem. The use of freezer is quite simple and well documented in Documentation/cgroups/freezer-subsystem.txt. The patch just uses it as it is. That is like: # freeze the container echo FROZEN > /target/container/freezer.state # unfreeze the container echo THAWED > /target/container/freezer.state However, freezer sometimes looks behaving different from the document. The document is saying "It's important to note that freezing can be incomplete. In that case we return EBUSY.", but in the case returning 0, freezing can be incomplete under load. So the code doesn't believe the return value and always check the content of freezer.state to identify whether the freezing is finished. Please refer to the comments in the code for more details. And the patch modifies domain_conf.c that assumes obj->monitor_chr != NULL but that is true only when qemu. Thanks, ozaki-r From: Ryota Ozaki <ozaki.ryota(a)gmail.com&gt; Date: Wed, 16 Sep 2009 17:59:54 +0900 Subject: [PATCH] lxc: suspend/resume support --- src/cgroup.c | 23 ++++++- src/cgroup.h | 4 + src/domain_conf.c | 27 ++++---- src/lxc_driver.c | 186 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 225 insertions(+), 15 deletions(-) diff --git a/src/cgroup.c b/src/cgroup.c index 111601d..2e646fd 100644 --- a/src/cgroup.c +++ b/src/cgroup.c @@ -32,7 +32,8 @@ #define CGROUP_MAX_VAL 512 VIR_ENUM_IMPL(virCgroupController, VIR_CGROUP_CONTROLLER_LAST, - "cpu", "cpuacct", "cpuset", "memory", "devices"); + "cpu", "cpuacct", "cpuset", "memory", "devices", + "freezer"); struct virCgroupController { int type; @@ -896,3 +897,23 @@ int virCgroupGetCpuacctUsage(virCgroupPtr group, unsigned long long *usage) VIR_CGROUP_CONTROLLER_CPUACCT, "cpuacct.usage", (uint64_t *)usage); } + +int virCgroupSetFreezerState(virCgroupPtr group, const char *state) +{ + return virCgroupSetValueStr(group, + VIR_CGROUP_CONTROLLER_CPU, + "freezer.state", state); +} + +int virCgroupGetFreezerState(virCgroupPtr group, char **state) +{ + int ret; + ret = virCgroupGetValueStr(group, + VIR_CGROUP_CONTROLLER_CPU, + "freezer.state", state); + if (ret == 0) { + char *p = strchr(*state, '¥n'); + if (p) *p = '¥0'; + } + return ret; +} diff --git a/src/cgroup.h b/src/cgroup.h index 6d43b14..aba56c6 100644 --- a/src/cgroup.h +++ b/src/cgroup.h @@ -21,6 +21,7 @@ enum { VIR_CGROUP_CONTROLLER_CPUSET, VIR_CGROUP_CONTROLLER_MEMORY, VIR_CGROUP_CONTROLLER_DEVICES, + VIR_CGROUP_CONTROLLER_FREEZER, VIR_CGROUP_CONTROLLER_LAST }; @@ -68,6 +69,9 @@ int virCgroupGetCpuShares(virCgroupPtr group, unsigned long long *shares); int virCgroupGetCpuacctUsage(virCgroupPtr group, unsigned long long *usage); +int virCgroupSetFreezerState(virCgroupPtr group, const char *state); +int virCgroupGetFreezerState(virCgroupPtr group, char **state); + int virCgroupRemove(virCgroupPtr group); void virCgroupFree(virCgroupPtr *group); diff --git a/src/domain_conf.c b/src/domain_conf.c index 5ae0775..5e37d96 100644 --- a/src/domain_conf.c +++ b/src/domain_conf.c @@ -4433,19 +4433,22 @@ char *virDomainObjFormat(virConnectPtr conn, virDomainStateTypeToString(obj->state), obj->pid); - switch (obj->monitor_chr->type) { - case VIR_DOMAIN_CHR_TYPE_UNIX: - monitorpath = obj->monitor_chr->data.nix.path; - break; - default: - case VIR_DOMAIN_CHR_TYPE_PTY: - monitorpath = obj->monitor_chr->data.file.path; - break; - } + /* obj->monitor_chr is set only for qemu */ + if (obj->monitor_chr) { + switch (obj->monitor_chr->type) { + case VIR_DOMAIN_CHR_TYPE_UNIX: + monitorpath = obj->monitor_chr->data.nix.path; + break; + default: + case VIR_DOMAIN_CHR_TYPE_PTY: + monitorpath = obj->monitor_chr->data.file.path; + break; + } - virBufferEscapeString(&buf, " <monitor path='%s'", monitorpath); - virBufferVSprintf(&buf, " type='%s'/>¥n", - virDomainChrTypeToString(obj->monitor_chr->type)); + virBufferEscapeString(&buf, " <monitor path='%s'", monitorpath); + virBufferVSprintf(&buf, " type='%s'/>¥n", + virDomainChrTypeToString(obj->monitor_chr->type)); + } if (obj->nvcpupids) { diff --git a/src/lxc_driver.c b/src/lxc_driver.c index 0ec1e92..2399d98 100644 --- a/src/lxc_driver.c +++ b/src/lxc_driver.c @@ -1862,6 +1862,188 @@ static char *lxcGetHostname (virConnectPtr conn) return result; } +static int lxcFreezeContainer(lxc_driver_t *driver, virDomainObjPtr vm) +{ + int timeout = 3; /* In seconds */ + int check_interval = 500; /* In milliseconds */ + int n_try = (timeout * (1000 / check_interval)); + int i = 0; + int ret = -1; + char *state = NULL; + virCgroupPtr cgroup = NULL; + + if (!(driver->cgroup && + virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) == 0)) + return -1; + + while (++i <= n_try) { + int r; + /* + * Writing "FROZEN" to the "freezer.state" freezes the group, + * i.e., the container, temporarily transiting "FREEZING" state. + * Once the freezing is completed, the state of the group transits + * to "FROZEN". + * (see linux-2.6/Documentation/cgroups/freezer-subsystem.txt) + */ + r = virCgroupSetFreezerState(cgroup, "FROZEN"); + + /* + * Returning EBUSY explicitly indicates that the group is + * being freezed but incomplete and other errors are true + * errors. + */ + if (r < 0 && r != -EBUSY) { + VIR_DEBUG("Writing freezer.state failed with errno: %d", r); + goto error; + } + + /* + * Unfortunately, returning 0 (success) is likely to happen + * even when the freezing has not been completed. Sometimes + * the state of the group remains "FREEZING" like when + * returning -EBUSY and even worse may never transit to + * "FROZEN" even if writing "FROZEN" again. + * + * So we don't trust the return value anyway and always + * decide that the freezing has been complete only with + * the state actually transit to "FROZEN". + */ + usleep(check_interval * 1000); + + r = virCgroupGetFreezerState(cgroup, &state); + + if (r < 0) { + VIR_DEBUG("Reading freezer.state failed with errno: %d", r); + goto error; + } + VIR_DEBUG("Read freezer.state: %s", state); + + if (STREQ(state, "FROZEN")) { + ret = 0; + goto cleanup; + } + + VIR_FREE(state); + } + VIR_DEBUG0("lxcFreezeContainer timeout"); +error: + /* + * If timeout or an error on reading the state occurs, + * activate the group again and return an error. + * This is likely to fall the group back again gracefully. + */ + virCgroupSetFreezerState(cgroup, "THAWED"); + ret = -1; + +cleanup: + if (cgroup) + virCgroupFree(&cgroup); + VIR_FREE(state); + return ret; +} + +static int lxcDomainSuspend(virDomainPtr dom) +{ + lxc_driver_t *driver = dom->conn->privateData; + virDomainObjPtr vm; + int ret = -1; + + lxcDriverLock(driver); + vm = virDomainFindByUUID(&driver->domains, dom->uuid); + + if (!vm) { + char uuidstr[VIR_UUID_STRING_BUFLEN]; + virUUIDFormat(dom->uuid, uuidstr); + lxcError(dom->conn, dom, VIR_ERR_NO_DOMAIN, + _("no domain with matching uuid '%s'"), uuidstr); + goto cleanup; + } + + if (!virDomainIsActive(vm)) { + lxcError(dom->conn, dom, VIR_ERR_OPERATION_INVALID, + "%s", _("domain is not running")); + goto cleanup; + } + + if (vm->state != VIR_DOMAIN_PAUSED) { + if (lxcFreezeContainer(driver, vm) < 0) { + lxcError(dom->conn, dom, VIR_ERR_OPERATION_FAILED, + "%s", _("suspend operation failed")); + goto cleanup; + } + vm->state = VIR_DOMAIN_PAUSED; + } + + if (virDomainSaveStatus(dom->conn, driver->stateDir, vm) < 0) + goto cleanup; + ret = 0; + +cleanup: + if (vm) + virDomainObjUnlock(vm); + lxcDriverUnlock(driver); + return ret; +} + +static int lxcUnfreezeContainer(lxc_driver_t *driver, virDomainObjPtr vm) +{ + int ret; + virCgroupPtr cgroup = NULL; + + if (!(driver->cgroup && + virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0) == 0)) + return -1; + + ret = virCgroupSetFreezerState(cgroup, "THAWED"); + + virCgroupFree(&cgroup); + return ret; +} + +static int lxcDomainResume(virDomainPtr dom) +{ + lxc_driver_t *driver = dom->conn->privateData; + virDomainObjPtr vm; + int ret = -1; + + lxcDriverLock(driver); + vm = virDomainFindByUUID(&driver->domains, dom->uuid); + + if (!vm) { + char uuidstr[VIR_UUID_STRING_BUFLEN]; + virUUIDFormat(dom->uuid, uuidstr); + lxcError(dom->conn, dom, VIR_ERR_NO_DOMAIN, + _("no domain with matching uuid '%s'"), uuidstr); + goto cleanup; + } + + if (!virDomainIsActive(vm)) { + lxcError(dom->conn, dom, VIR_ERR_OPERATION_INVALID, + "%s", _("domain is not running")); + goto cleanup; + } + + if (vm->state == VIR_DOMAIN_PAUSED) { + if (lxcUnfreezeContainer(driver, vm) < 0) { + lxcError(dom->conn, dom, VIR_ERR_OPERATION_FAILED, + "%s", _("resume operation failed")); + goto cleanup; + } + vm->state = VIR_DOMAIN_RUNNING; + } + + if (virDomainSaveStatus(dom->conn, driver->stateDir, vm) < 0) + goto cleanup; + ret = 0; + +cleanup: + if (vm) + virDomainObjUnlock(vm); + lxcDriverUnlock(driver); + return ret; +} + + /* Function Tables */ static virDriver lxcDriver = { VIR_DRV_LXC, /* the number virDrvNo */ @@ -1881,8 +2063,8 @@ static virDriver lxcDriver = { lxcDomainLookupByID, /* domainLookupByID */ lxcDomainLookupByUUID, /* domainLookupByUUID */ lxcDomainLookupByName, /* domainLookupByName */ - NULL, /* domainSuspend */ - NULL, /* domainResume */ + lxcDomainSuspend, /* domainSuspend */ + lxcDomainResume, /* domainResume */ lxcDomainShutdown, /* domainShutdown */ NULL, /* domainReboot */ lxcDomainDestroy, /* domainDestroy */ -- 1.6.0.6