This patch adds initial migration support to the libxl driver, using the VIR_DRV_FEATURE_MIGRATION_PARAMS family of migration functions.

Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; --- V6 here https://www.redhat.com/archives/libvir-list/2014-May/msg00017.html In V7: There were no comments on V6, but during testing I noticed that 'virsh migrate --suspend ...' was not being honored. Fixed that in this version so that the domain is paused on the destination once migration completes. po/POTFILES.in | 1 + src/Makefile.am | 3 +- src/libxl/libxl_conf.h | 6 + src/libxl/libxl_domain.h | 1 + src/libxl/libxl_driver.c | 235 ++++++++++++++++++ src/libxl/libxl_migration.c | 581 ++++++++++++++++++++++++++++++++++++++++++++ src/libxl/libxl_migration.h | 79 ++++++ 7 files changed, 905 insertions(+), 1 deletion(-) diff --git a/po/POTFILES.in b/po/POTFILES.in index e35eb82..a72dc1e 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -73,6 +73,7 @@ src/lxc/lxc_process.c src/libxl/libxl_domain.c src/libxl/libxl_driver.c src/libxl/libxl_conf.c +src/libxl/libxl_migration.c src/network/bridge_driver.c src/network/bridge_driver_linux.c src/node_device/node_device_driver.c diff --git a/src/Makefile.am b/src/Makefile.am index e9dc9e0..0dbda7f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -706,7 +706,8 @@ XENAPI_DRIVER_SOURCES = \ LIBXL_DRIVER_SOURCES = \ libxl/libxl_conf.c libxl/libxl_conf.h \ libxl/libxl_domain.c libxl/libxl_domain.h \ - libxl/libxl_driver.c libxl/libxl_driver.h + libxl/libxl_driver.c libxl/libxl_driver.h \ + libxl/libxl_migration.c libxl/libxl_migration.h UML_DRIVER_SOURCES = \ uml/uml_conf.c uml/uml_conf.h \ diff --git a/src/libxl/libxl_conf.h b/src/libxl/libxl_conf.h index 24e1720..b798567 100644 --- a/src/libxl/libxl_conf.h +++ b/src/libxl/libxl_conf.h @@ -43,6 +43,9 @@ # define LIBXL_VNC_PORT_MIN 5900 # define LIBXL_VNC_PORT_MAX 65535 +# define LIBXL_MIGRATION_PORT_MIN 49152 +# define LIBXL_MIGRATION_PORT_MAX 49216 + # define LIBXL_CONFIG_DIR SYSCONFDIR "/libvirt/libxl" # define LIBXL_AUTOSTART_DIR LIBXL_CONFIG_DIR "/autostart" # define LIBXL_STATE_DIR LOCALSTATEDIR "/run/libvirt/libxl" @@ -115,6 +118,9 @@ struct _libxlDriverPrivate { /* Immutable pointer, self-locking APIs */ virPortAllocatorPtr reservedVNCPorts; + /* Immutable pointer, self-locking APIs */ + virPortAllocatorPtr migrationPorts; + /* Immutable pointer, lockless APIs*/ virSysinfoDefPtr hostsysinfo; }; diff --git a/src/libxl/libxl_domain.h b/src/libxl/libxl_domain.h index 979ce2a..9d48049 100644 --- a/src/libxl/libxl_domain.h +++ b/src/libxl/libxl_domain.h @@ -69,6 +69,7 @@ struct _libxlDomainObjPrivate { virChrdevsPtr devs; libxl_evgen_domain_death *deathW; libxlDriverPrivatePtr driver; + unsigned short migrationPort; struct libxlDomainJobObj job; }; diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c index df7d510..35467ac 100644 --- a/src/libxl/libxl_driver.c +++ b/src/libxl/libxl_driver.c @@ -45,6 +45,7 @@ #include "libxl_domain.h" #include "libxl_driver.h" #include "libxl_conf.h" +#include "libxl_migration.h" #include "xen_xm.h" #include "xen_sxpr.h" #include "virtypedparam.h" @@ -209,6 +210,7 @@ libxlStateCleanup(void) virObjectUnref(libxl_driver->xmlopt); virObjectUnref(libxl_driver->domains); virObjectUnref(libxl_driver->reservedVNCPorts); + virObjectUnref(libxl_driver->migrationPorts); virObjectEventStateFree(libxl_driver->domainEventState); virSysinfoDefFree(libxl_driver->hostsysinfo); @@ -301,6 +303,13 @@ libxlStateInitialize(bool privileged, LIBXL_VNC_PORT_MAX))) goto error; + /* Allocate bitmap for migration port reservation */ + if (!(libxl_driver->migrationPorts = + virPortAllocatorNew(_("migration"), + LIBXL_MIGRATION_PORT_MIN, + LIBXL_MIGRATION_PORT_MAX))) + goto error; + if (!(libxl_driver->domains = virDomainObjListNew())) goto error; @@ -4155,6 +4164,7 @@ libxlConnectSupportsFeature(virConnectPtr conn, int feature) switch (feature) { case VIR_DRV_FEATURE_TYPED_PARAM_STRING: + case VIR_DRV_FEATURE_MIGRATION_PARAMS: return 1; default: return 0; @@ -4333,6 +4343,226 @@ libxlNodeDeviceReset(virNodeDevicePtr dev) return ret; } +static char * +libxlDomainMigrateBegin3Params(virDomainPtr domain, + virTypedParameterPtr params, + int nparams, + char **cookieout ATTRIBUTE_UNUSED, + int *cookieoutlen ATTRIBUTE_UNUSED, + unsigned int flags) +{ + const char *xmlin = NULL; + virDomainObjPtr vm = NULL; + + virCheckFlags(LIBXL_MIGRATION_FLAGS, NULL); + if (virTypedParamsValidate(params, nparams, LIBXL_MIGRATION_PARAMETERS) < 0) + return NULL; + + if (virTypedParamsGetString(params, nparams, + VIR_MIGRATE_PARAM_DEST_XML, + &xmlin) < 0) + return NULL; + + if (!(vm = libxlDomObjFromDomain(domain))) + return NULL; + + if (virDomainMigrateBegin3ParamsEnsureACL(domain->conn, vm->def) < 0) { + virObjectUnlock(vm); + return NULL; + } + + if (!virDomainObjIsActive(vm)) { + virReportError(VIR_ERR_OPERATION_INVALID, + "%s", _("domain is not running")); + virObjectUnlock(vm); + return NULL; + } + + return libxlDomainMigrationBegin(domain->conn, vm, xmlin); +} + +static int +libxlDomainMigratePrepare3Params(virConnectPtr dconn, + virTypedParameterPtr params, + int nparams, + const char *cookiein ATTRIBUTE_UNUSED, + int cookieinlen ATTRIBUTE_UNUSED, + char **cookieout ATTRIBUTE_UNUSED, + int *cookieoutlen ATTRIBUTE_UNUSED, + char **uri_out, + unsigned int flags) +{ + libxlDriverPrivatePtr driver = dconn->privateData; + virDomainDefPtr def = NULL; + const char *dom_xml = NULL; + const char *dname = NULL; + const char *uri_in = NULL; + + virCheckFlags(LIBXL_MIGRATION_FLAGS, -1); + if (virTypedParamsValidate(params, nparams, LIBXL_MIGRATION_PARAMETERS) < 0) + goto error; + + if (virTypedParamsGetString(params, nparams, + VIR_MIGRATE_PARAM_DEST_XML, + &dom_xml) < 0 || + virTypedParamsGetString(params, nparams, + VIR_MIGRATE_PARAM_DEST_NAME, + &dname) < 0 || + virTypedParamsGetString(params, nparams, + VIR_MIGRATE_PARAM_URI, + &uri_in) < 0) + + goto error; + + if (!(def = libxlDomainMigrationPrepareDef(driver, dom_xml, dname))) + goto error; + + if (virDomainMigratePrepare3ParamsEnsureACL(dconn, def) < 0) + goto error; + + if (libxlDomainMigrationPrepare(dconn, def, uri_in, uri_out, flags) < 0) + goto error; + + return 0; + + error: + virDomainDefFree(def); + return -1; +} + +static int +libxlDomainMigratePerform3Params(virDomainPtr dom, + const char *dconnuri, + virTypedParameterPtr params, + int nparams, + const char *cookiein ATTRIBUTE_UNUSED, + int cookieinlen ATTRIBUTE_UNUSED, + char **cookieout ATTRIBUTE_UNUSED, + int *cookieoutlen ATTRIBUTE_UNUSED, + unsigned int flags) +{ + libxlDriverPrivatePtr driver = dom->conn->privateData; + virDomainObjPtr vm = NULL; + const char *dom_xml = NULL; + const char *dname = NULL; + const char *uri = NULL; + int ret = -1; + + virCheckFlags(LIBXL_MIGRATION_FLAGS, -1); + if (virTypedParamsValidate(params, nparams, LIBXL_MIGRATION_PARAMETERS) < 0) + goto cleanup; + + if (virTypedParamsGetString(params, nparams, + VIR_MIGRATE_PARAM_DEST_XML, + &dom_xml) < 0 || + virTypedParamsGetString(params, nparams, + VIR_MIGRATE_PARAM_DEST_NAME, + &dname) < 0 || + virTypedParamsGetString(params, nparams, + VIR_MIGRATE_PARAM_URI, + &uri) < 0) + + goto cleanup; + + if (!(vm = libxlDomObjFromDomain(dom))) + goto cleanup; + + if (virDomainMigratePerform3ParamsEnsureACL(dom->conn, vm->def) < 0) + goto cleanup; + + if (libxlDomainMigrationPerform(driver, vm, dom_xml, dconnuri, + uri, dname, flags) < 0) { + /* Job terminated and vm unlocked if MigrationPerform failed */ + vm = NULL; + goto cleanup; + } + + ret = 0; + + cleanup: + if (vm) + virObjectUnlock(vm); + return ret; +} + +static virDomainPtr +libxlDomainMigrateFinish3Params(virConnectPtr dconn, + virTypedParameterPtr params, + int nparams, + const char *cookiein ATTRIBUTE_UNUSED, + int cookieinlen ATTRIBUTE_UNUSED, + char **cookieout ATTRIBUTE_UNUSED, + int *cookieoutlen ATTRIBUTE_UNUSED, + unsigned int flags, + int cancelled) +{ + libxlDriverPrivatePtr driver = dconn->privateData; + virDomainObjPtr vm = NULL; + const char *dname = NULL; + + virCheckFlags(LIBXL_MIGRATION_FLAGS, NULL); + if (virTypedParamsValidate(params, nparams, LIBXL_MIGRATION_PARAMETERS) < 0) + return NULL; + + if (virTypedParamsGetString(params, nparams, + VIR_MIGRATE_PARAM_DEST_NAME, + &dname) < 0) + return NULL; + + if (!dname || + !(vm = virDomainObjListFindByName(driver->domains, dname))) { + /* Migration obviously failed if the domain doesn't exist */ + virReportError(VIR_ERR_OPERATION_FAILED, + _("Migration failed. No domain on destination host " + "with matching name '%s'"), + NULLSTR(dname)); + return NULL; + } + + if (virDomainMigrateFinish3ParamsEnsureACL(dconn, vm->def) < 0) { + virObjectUnlock(vm); + return NULL; + } + + if (!virDomainObjIsActive(vm)) { + /* Migration failed if domain is inactive */ + virReportError(VIR_ERR_OPERATION_FAILED, + "%s", _("Migration failed. Domain is not running " + "on destination host")); + virObjectUnlock(vm); + return NULL; + } + + return libxlDomainMigrationFinish(dconn, vm, flags, cancelled); +} + +static int +libxlDomainMigrateConfirm3Params(virDomainPtr domain, + virTypedParameterPtr params, + int nparams, + const char *cookiein ATTRIBUTE_UNUSED, + int cookieinlen ATTRIBUTE_UNUSED, + unsigned int flags, + int cancelled) +{ + libxlDriverPrivatePtr driver = domain->conn->privateData; + virDomainObjPtr vm = NULL; + + virCheckFlags(LIBXL_MIGRATION_FLAGS, -1); + if (virTypedParamsValidate(params, nparams, LIBXL_MIGRATION_PARAMETERS) < 0) + return -1; + + if (!(vm = libxlDomObjFromDomain(domain))) + return -1; + + if (virDomainMigrateConfirm3ParamsEnsureACL(domain->conn, vm->def) < 0) { + virObjectUnlock(vm); + return -1; + } + + return libxlDomainMigrationConfirm(driver, vm, flags, cancelled); +} + static virDriver libxlDriver = { .no = VIR_DRV_LIBXL, @@ -4423,6 +4653,11 @@ static virDriver libxlDriver = { .nodeDeviceDetachFlags = libxlNodeDeviceDetachFlags, /* 1.2.3 */ .nodeDeviceReAttach = libxlNodeDeviceReAttach, /* 1.2.3 */ .nodeDeviceReset = libxlNodeDeviceReset, /* 1.2.3 */ + .domainMigrateBegin3Params = libxlDomainMigrateBegin3Params, /* 1.2.3 */ + .domainMigratePrepare3Params = libxlDomainMigratePrepare3Params, /* 1.2.3 */ + .domainMigratePerform3Params = libxlDomainMigratePerform3Params, /* 1.2.3 */ + .domainMigrateFinish3Params = libxlDomainMigrateFinish3Params, /* 1.2.3 */ + .domainMigrateConfirm3Params = libxlDomainMigrateConfirm3Params, /* 1.2.3 */ }; static virStateDriver libxlStateDriver = { diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c new file mode 100644 index 0000000..e3699c5 --- /dev/null +++ b/src/libxl/libxl_migration.c @@ -0,0 +1,581 @@ +/* + * libxl_migration.c: methods for handling migration with libxenlight + * + * Copyright (C) 2014 SUSE LINUX Products GmbH, Nuernberg, Germany. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Jim Fehlig <jfehlig(a)suse.com&gt; + * Chunyan Liu <cyliu(a)suse.com&gt; + */ + +#include <config.h> + +#include "internal.h" +#include "virlog.h" +#include "virerror.h" +#include "virconf.h" +#include "datatypes.h" +#include "virfile.h" +#include "viralloc.h" +#include "viruuid.h" +#include "vircommand.h" +#include "virstring.h" +#include "virobject.h" +#include "rpc/virnetsocket.h" +#include "libxl_domain.h" +#include "libxl_driver.h" +#include "libxl_conf.h" +#include "libxl_migration.h" + +#define VIR_FROM_THIS VIR_FROM_LIBXL + +VIR_LOG_INIT("libxl.libxl_migration"); + +typedef struct _libxlMigrationDstArgs { + virObject parent; + + virConnectPtr conn; + virDomainObjPtr vm; + unsigned int flags; + + /* for freeing listen sockets */ + virNetSocketPtr *socks; + size_t nsocks; +} libxlMigrationDstArgs; + +static virClassPtr libxlMigrationDstArgsClass; + +static void +libxlMigrationDstArgsDispose(void *obj) +{ + libxlMigrationDstArgs *args = obj; + + VIR_FREE(args->socks); +} + +static int +libxlMigrationDstArgsOnceInit(void) +{ + if (!(libxlMigrationDstArgsClass = virClassNew(virClassForObject(), + "libxlMigrationDstArgs", + sizeof(libxlMigrationDstArgs), + libxlMigrationDstArgsDispose))) + return -1; + + return 0; +} + +VIR_ONCE_GLOBAL_INIT(libxlMigrationDstArgs) + +static void +libxlDoMigrateReceive(virNetSocketPtr sock, + int events ATTRIBUTE_UNUSED, + void *opaque) +{ + libxlMigrationDstArgs *args = opaque; + virConnectPtr conn = args->conn; + virDomainObjPtr vm = args->vm; + virNetSocketPtr *socks = args->socks; + size_t nsocks = args->nsocks; + bool paused = args->flags & VIR_MIGRATE_PAUSED; + libxlDriverPrivatePtr driver = conn->privateData; + virNetSocketPtr client_sock; + int recvfd = -1; + size_t i; + int ret; + + virNetSocketAccept(sock, &client_sock); + if (client_sock == NULL) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("Fail to accept migration connection")); + goto cleanup; + } + VIR_DEBUG("Accepted migration connection\n"); + recvfd = virNetSocketDupFD(client_sock, true); + virObjectUnref(client_sock); + + virObjectLock(vm); + ret = libxlDomainStart(driver, vm, paused, recvfd); + virObjectUnlock(vm); + + if (ret < 0 && !vm->persistent) + virDomainObjListRemove(driver->domains, vm); + + cleanup: + /* Remove all listen socks from event handler, and close them. */ + for (i = 0; i < nsocks; i++) { + virNetSocketUpdateIOCallback(socks[i], 0); + virNetSocketRemoveIOCallback(socks[i]); + virNetSocketClose(socks[i]); + virObjectUnref(socks[i]); + } + + VIR_FORCE_CLOSE(recvfd); +} + +static int +libxlDoMigrateSend(libxlDriverPrivatePtr driver, + virDomainObjPtr vm, + unsigned long flags, + int sockfd) +{ + libxlDomainObjPrivatePtr priv; + libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver); + virObjectEventPtr event = NULL; + int xl_flags = 0; + int ret; + + if (flags & VIR_MIGRATE_LIVE) + xl_flags = LIBXL_SUSPEND_LIVE; + + priv = vm->privateData; + ret = libxl_domain_suspend(priv->ctx, vm->def->id, sockfd, + xl_flags, NULL); + if (ret != 0) { + /* attempt to resume the domain on failure */ + if (libxl_domain_resume(priv->ctx, vm->def->id, 0, 0) != 0) { + VIR_DEBUG("Failed to resume domain following failed migration"); + virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, + VIR_DOMAIN_PAUSED_MIGRATION); + event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED); + ignore_value(virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm)); + } + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to send migration data to destination host")); + ret = -1; + goto cleanup; + } + + cleanup: + if (event) + libxlDomainEventQueue(driver, event); + virObjectUnref(cfg); + return ret; +} + +static bool +libxlDomainMigrationIsAllowed(virDomainDefPtr def) +{ + /* Migration is not allowed if definition contains any hostdevs */ + if (def->nhostdevs > 0) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("domain has assigned host devices")); + return false; + } + + return true; +} + +char * +libxlDomainMigrationBegin(virConnectPtr conn, + virDomainObjPtr vm, + const char *xmlin) +{ + libxlDriverPrivatePtr driver = conn->privateData; + libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver); + virDomainDefPtr tmpdef = NULL; + virDomainDefPtr def; + char *xml = NULL; + + if (libxlDomainObjBeginJob(driver, vm, LIBXL_JOB_MODIFY) < 0) + goto cleanup; + + if (xmlin) { + if (!(tmpdef = virDomainDefParseString(xmlin, cfg->caps, + driver->xmlopt, + 1 << VIR_DOMAIN_VIRT_XEN, + VIR_DOMAIN_XML_INACTIVE))) + goto endjob; + + def = tmpdef; + } else { + def = vm->def; + } + + if (!libxlDomainMigrationIsAllowed(def)) + goto endjob; + + xml = virDomainDefFormat(def, VIR_DOMAIN_XML_SECURE); + + cleanup: + if (vm) + virObjectUnlock(vm); + + virDomainDefFree(tmpdef); + virObjectUnref(cfg); + return xml; + + endjob: + if (!libxlDomainObjEndJob(driver, vm)) + vm = NULL; + goto cleanup; +} + +virDomainDefPtr +libxlDomainMigrationPrepareDef(libxlDriverPrivatePtr driver, + const char *dom_xml, + const char *dname) +{ + libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver); + virDomainDefPtr def; + char *name = NULL; + + if (!dom_xml) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("no domain XML passed")); + return NULL; + } + + if (!(def = virDomainDefParseString(dom_xml, cfg->caps, driver->xmlopt, + 1 << VIR_DOMAIN_VIRT_XEN, + VIR_DOMAIN_XML_INACTIVE))) + goto cleanup; + + if (dname) { + name = def->name; + if (VIR_STRDUP(def->name, dname) < 0) { + virDomainDefFree(def); + def = NULL; + } + } + + cleanup: + virObjectUnref(cfg); + VIR_FREE(name); + return def; +} + +int +libxlDomainMigrationPrepare(virConnectPtr dconn, + virDomainDefPtr def, + const char *uri_in, + char **uri_out, + unsigned int flags) +{ + libxlDriverPrivatePtr driver = dconn->privateData; + virDomainObjPtr vm = NULL; + char *hostname = NULL; + unsigned short port; + char portstr[100]; + virURIPtr uri = NULL; + virNetSocketPtr *socks = NULL; + size_t nsocks = 0; + int nsocks_listen = 0; + libxlMigrationDstArgs *args; + size_t i; + int ret = -1; + + if (!(vm = virDomainObjListAdd(driver->domains, def, + driver->xmlopt, + VIR_DOMAIN_OBJ_LIST_ADD_LIVE | + VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE, + NULL))) + goto cleanup; + + /* Create socket connection to receive migration data */ + if (!uri_in) { + if ((hostname = virGetHostname()) == NULL) + goto cleanup; + + if (STRPREFIX(hostname, "localhost")) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("hostname on destination resolved to localhost," + " but migration requires an FQDN")); + goto cleanup; + } + + if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) + goto cleanup; + + if (virAsprintf(uri_out, "tcp://%s:%d", hostname, port) < 0) + goto cleanup; + } else { + if (!(STRPREFIX(uri_in, "tcp://"))) { + /* not full URI, add prefix tcp:// */ + char *tmp; + if (virAsprintf(&tmp, "tcp://%s", uri_in) < 0) + goto cleanup; + uri = virURIParse(tmp); + VIR_FREE(tmp); + } else { + uri = virURIParse(uri_in); + } + + if (uri == NULL) { + virReportError(VIR_ERR_INVALID_ARG, + _("unable to parse URI: %s"), + uri_in); + goto cleanup; + } + + if (uri->server == NULL) { + virReportError(VIR_ERR_INVALID_ARG, + _("missing host in migration URI: %s"), + uri_in); + goto cleanup; + } else { + hostname = uri->server; + } + + if (uri->port == 0) { + if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) + goto cleanup; + + } else { + port = uri->port; + } + + if (virAsprintf(uri_out, "tcp://%s:%d", hostname, port) < 0) + goto cleanup; + } + + snprintf(portstr, sizeof(portstr), "%d", port); + + if (virNetSocketNewListenTCP(hostname, portstr, &socks, &nsocks) < 0) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("Fail to create socket for incoming migration")); + goto cleanup; + } + + if (libxlMigrationDstArgsInitialize() < 0) + goto cleanup; + + if (!(args = virObjectNew(libxlMigrationDstArgsClass))) + goto cleanup; + + args->conn = dconn; + args->vm = vm; + args->flags = flags; + args->socks = socks; + args->nsocks = nsocks; + + for (i = 0; i < nsocks; i++) { + if (virNetSocketSetBlocking(socks[i], true) < 0) + continue; + + if (virNetSocketListen(socks[i], 1) < 0) + continue; + + if (virNetSocketAddIOCallback(socks[i], + VIR_EVENT_HANDLE_READABLE, + libxlDoMigrateReceive, + args, + virObjectFreeCallback) < 0) + continue; + + /* + * Successfully added sock to event loop. Take a ref on args to + * ensure it is not freed until sock is removed from the event loop. + * Ref is dropped in virObjectFreeCallback after being removed + * from the event loop. + */ + virObjectRef(args); + nsocks_listen++; + } + + /* Done with args in this function, drop reference */ + virObjectUnref(args); + + if (!nsocks_listen) + goto cleanup; + + ret = 0; + goto done; + + cleanup: + for (i = 0; i < nsocks; i++) { + virNetSocketClose(socks[i]); + virObjectUnref(socks[i]); + } + + done: + virURIFree(uri); + if (vm) + virObjectUnlock(vm); + return ret; +} + +int +libxlDomainMigrationPerform(libxlDriverPrivatePtr driver, + virDomainObjPtr vm, + const char *dom_xml ATTRIBUTE_UNUSED, + const char *dconnuri ATTRIBUTE_UNUSED, + const char *uri_str, + const char *dname ATTRIBUTE_UNUSED, + unsigned int flags) +{ + char *hostname = NULL; + unsigned short port = 0; + char portstr[100]; + virURIPtr uri = NULL; + virNetSocketPtr sock; + int sockfd = -1; + int saved_errno = EINVAL; + int ret = -1; + + /* parse dst host:port from uri */ + uri = virURIParse(uri_str); + if (uri == NULL || uri->server == NULL || uri->port == 0) + goto cleanup; + + hostname = uri->server; + port = uri->port; + snprintf(portstr, sizeof(portstr), "%d", port); + + /* socket connect to dst host:port */ + if (virNetSocketNewConnectTCP(hostname, portstr, &sock) < 0) { + virReportSystemError(saved_errno, + _("unable to connect to '%s:%s'"), + hostname, portstr); + goto cleanup; + } + + if (virNetSocketSetBlocking(sock, true) < 0) { + virObjectUnref(sock); + goto cleanup; + } + + sockfd = virNetSocketDupFD(sock, true); + virObjectUnref(sock); + + /* suspend vm and send saved data to dst through socket fd */ + virObjectUnlock(vm); + ret = libxlDoMigrateSend(driver, vm, flags, sockfd); + virObjectLock(vm); + + cleanup: + /* If failure, terminate the job started in MigrationBegin */ + if (ret == -1) { + if (libxlDomainObjEndJob(driver, vm)) + virObjectUnlock(vm); + } + VIR_FORCE_CLOSE(sockfd); + virURIFree(uri); + return ret; +} + +virDomainPtr +libxlDomainMigrationFinish(virConnectPtr dconn, + virDomainObjPtr vm, + unsigned int flags, + int cancelled) +{ + libxlDriverPrivatePtr driver = dconn->privateData; + libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver); + libxlDomainObjPrivatePtr priv = vm->privateData; + virObjectEventPtr event = NULL; + virDomainPtr dom = NULL; + + virPortAllocatorRelease(driver->migrationPorts, priv->migrationPort); + priv->migrationPort = 0; + + if (cancelled) + goto cleanup; + + if (!(flags & VIR_MIGRATE_PAUSED)) { + if (libxl_domain_unpause(priv->ctx, vm->def->id) != 0) { + virReportError(VIR_ERR_OPERATION_FAILED, "%s", + _("Failed to unpause domain")); + goto cleanup; + } + + virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, + VIR_DOMAIN_RUNNING_MIGRATED); + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_RESUMED, + VIR_DOMAIN_EVENT_RESUMED_MIGRATED); + } else { + virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, VIR_DOMAIN_PAUSED_USER); + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_PAUSED); + } + + if (virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm) < 0) + goto cleanup; + + dom = virGetDomain(dconn, vm->def->name, vm->def->uuid); + + if (dom == NULL) { + libxl_domain_destroy(priv->ctx, vm->def->id, NULL); + libxlDomainCleanup(driver, vm, VIR_DOMAIN_SHUTOFF_FAILED); + event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_FAILED); + libxlDomainEventQueue(driver, event); + } + + cleanup: + if (event) + libxlDomainEventQueue(driver, event); + if (vm) + virObjectUnlock(vm); + virObjectUnref(cfg); + return dom; +} + +int +libxlDomainMigrationConfirm(libxlDriverPrivatePtr driver, + virDomainObjPtr vm, + unsigned int flags, + int cancelled) +{ + libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver); + libxlDomainObjPrivatePtr priv = vm->privateData; + virObjectEventPtr event = NULL; + int ret = -1; + + if (cancelled) { + if (libxl_domain_resume(priv->ctx, vm->def->id, 1, 0) == 0) { + ret = 0; + } else { + VIR_DEBUG("Unable to resume domain '%s' after failed migration", + vm->def->name); + virDomainObjSetState(vm, VIR_DOMAIN_PAUSED, + VIR_DOMAIN_PAUSED_MIGRATION); + event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED); + ignore_value(virDomainSaveStatus(driver->xmlopt, cfg->stateDir, vm)); + } + goto cleanup; + } + + libxl_domain_destroy(priv->ctx, vm->def->id, NULL); + libxlDomainCleanup(driver, vm, VIR_DOMAIN_SHUTOFF_MIGRATED); + event = virDomainEventLifecycleNewFromObj(vm, VIR_DOMAIN_EVENT_STOPPED, + VIR_DOMAIN_EVENT_STOPPED_MIGRATED); + + VIR_DEBUG("Domain '%s' successfully migrated", vm->def->name); + + if (flags & VIR_MIGRATE_UNDEFINE_SOURCE) + virDomainDeleteConfig(cfg->configDir, cfg->autostartDir, vm); + + if (!vm->persistent || (flags & VIR_MIGRATE_UNDEFINE_SOURCE)) + virDomainObjListRemove(driver->domains, vm); + + ret = 0; + + cleanup: + if (!libxlDomainObjEndJob(driver, vm)) + vm = NULL; + if (event) + libxlDomainEventQueue(driver, event); + if (vm) + virObjectUnlock(vm); + virObjectUnref(cfg); + return ret; +} diff --git a/src/libxl/libxl_migration.h b/src/libxl/libxl_migration.h new file mode 100644 index 0000000..aab96f5 --- /dev/null +++ b/src/libxl/libxl_migration.h @@ -0,0 +1,79 @@ +/* + * libxl_migration.h: methods for handling migration with libxenlight + * + * Copyright (c) 2014 SUSE LINUX Products GmbH, Nuernberg, Germany. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + * + * Authors: + * Jim Fehlig <jfehlig(a)suse.com&gt; + */ + +#ifndef LIBXL_MIGRATION_H +# define LIBXL_MIGRATION_H + +# include "libxl_conf.h" + +# define LIBXL_MIGRATION_FLAGS \ + (VIR_MIGRATE_LIVE | \ + VIR_MIGRATE_UNDEFINE_SOURCE | \ + VIR_MIGRATE_PAUSED) + +/* All supported migration parameters and their types. */ +# define LIBXL_MIGRATION_PARAMETERS \ + VIR_MIGRATE_PARAM_URI, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_DEST_NAME, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_DEST_XML, VIR_TYPED_PARAM_STRING, \ + NULL + +char * +libxlDomainMigrationBegin(virConnectPtr conn, + virDomainObjPtr vm, + const char *xmlin); + +virDomainDefPtr +libxlDomainMigrationPrepareDef(libxlDriverPrivatePtr driver, + const char *dom_xml, + const char *dname); + +int +libxlDomainMigrationPrepare(virConnectPtr dconn, + virDomainDefPtr def, + const char *uri_in, + char **uri_out, + unsigned int flags); + +int +libxlDomainMigrationPerform(libxlDriverPrivatePtr driver, + virDomainObjPtr vm, + const char *dom_xml, + const char *dconnuri, + const char *uri_str, + const char *dname, + unsigned int flags); + +virDomainPtr +libxlDomainMigrationFinish(virConnectPtr dconn, + virDomainObjPtr vm, + unsigned int flags, + int cancelled); + +int +libxlDomainMigrationConfirm(libxlDriverPrivatePtr driver, + virDomainObjPtr vm, + unsigned int flags, + int cancelled); + +#endif /* LIBXL_DRIVER_H */

On Thu, May 08, 2014 at 03:56:51PM -0600, Jim Fehlig wrote: > This patch adds initial migration support to the libxl driver, > using the VIR_DRV_FEATURE_MIGRATION_PARAMS family of migration > functions. > > Signed-off-by: Jim Fehlig <jfehlig(a)suse.com&gt; > --- > > V6 here > https://www.redhat.com/archives/libvir-list/2014-May/msg00017.html > > In V7: > There were no comments on V6, but during testing I noticed that > 'virsh migrate --suspend ...' was not being honored. Fixed that > in this version so that the domain is paused on the destination > once migration completes. > > po/POTFILES.in | 1 + > src/Makefile.am | 3 +- > src/libxl/libxl_conf.h | 6 + > src/libxl/libxl_domain.h | 1 + > src/libxl/libxl_driver.c | 235 ++++++++++++++++++ > src/libxl/libxl_migration.c | 581 ++++++++++++++++++++++++++++++++++++++++++++ > src/libxl/libxl_migration.h | 79 ++++++ > 7 files changed, 905 insertions(+), 1 deletion(-) > > @@ -301,6 +303,13 @@ libxlStateInitialize(bool privileged, > LIBXL_VNC_PORT_MAX))) > goto error; > > + /* Allocate bitmap for migration port reservation */ > + if (!(libxl_driver->migrationPorts = > + virPortAllocatorNew(_("migration"), > + LIBXL_MIGRATION_PORT_MIN, > + LIBXL_MIGRATION_PORT_MAX))) > + goto error; > + > No need todo it in this patch, but experiance with QEMU is that people will want this range to be configurable in /etc/libvirt/libxl.conf. So I'd suggest adding this later

> diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c > new file mode 100644 > index 0000000..e3699c5 > --- /dev/null > +++ b/src/libxl/libxl_migration.c > @@ -0,0 +1,581 @@ > +/* > + * libxl_migration.c: methods for handling migration with libxenlight > + * > + * Copyright (C) 2014 SUSE LINUX Products GmbH, Nuernberg, Germany. > + * > + * This library is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * This library is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with this library. If not, see > + * <http://www.gnu.org/licenses/>;. > + * > + * Authors: > + * Jim Fehlig <jfehlig(a)suse.com&gt; > + * Chunyan Liu <cyliu(a)suse.com&gt; > + */ > + > +#include <config.h> > + > +#include "internal.h" > +#include "virlog.h" > +#include "virerror.h" > +#include "virconf.h" > +#include "datatypes.h" > +#include "virfile.h" > +#include "viralloc.h" > +#include "viruuid.h" > +#include "vircommand.h" > +#include "virstring.h" > +#include "virobject.h" > +#include "rpc/virnetsocket.h" > +#include "libxl_domain.h" > +#include "libxl_driver.h" > +#include "libxl_conf.h" > +#include "libxl_migration.h" > + > +#define VIR_FROM_THIS VIR_FROM_LIBXL > + > +VIR_LOG_INIT("libxl.libxl_migration"); > + > +typedef struct _libxlMigrationDstArgs { > + virObject parent; > + > + virConnectPtr conn; > + virDomainObjPtr vm; > + unsigned int flags; > + > + /* for freeing listen sockets */ > + virNetSocketPtr *socks; > + size_t nsocks; > +} libxlMigrationDstArgs; > + > +static virClassPtr libxlMigrationDstArgsClass; > + > +static void > +libxlMigrationDstArgsDispose(void *obj) > +{ > + libxlMigrationDstArgs *args = obj; > + > + VIR_FREE(args->socks); > +} > + > +static int > +libxlMigrationDstArgsOnceInit(void) > +{ > + if (!(libxlMigrationDstArgsClass = virClassNew(virClassForObject(), > + "libxlMigrationDstArgs", > + sizeof(libxlMigrationDstArgs), > + libxlMigrationDstArgsDispose))) > + return -1; > + > + return 0; > +} > + > +VIR_ONCE_GLOBAL_INIT(libxlMigrationDstArgs) > + > +static void > +libxlDoMigrateReceive(virNetSocketPtr sock, > + int events ATTRIBUTE_UNUSED, > + void *opaque) > +{ > + libxlMigrationDstArgs *args = opaque; > + virConnectPtr conn = args->conn; > + virDomainObjPtr vm = args->vm; > + virNetSocketPtr *socks = args->socks; > + size_t nsocks = args->nsocks; > + bool paused = args->flags & VIR_MIGRATE_PAUSED; > + libxlDriverPrivatePtr driver = conn->privateData; > + virNetSocketPtr client_sock; > + int recvfd = -1; > + size_t i; > + int ret; > + > + virNetSocketAccept(sock, &client_sock); > + if (client_sock == NULL) { > + virReportError(VIR_ERR_OPERATION_INVALID, "%s", > + _("Fail to accept migration connection")); > + goto cleanup; > + } > + VIR_DEBUG("Accepted migration connection\n"); > + recvfd = virNetSocketDupFD(client_sock, true); > + virObjectUnref(client_sock); > + > + virObjectLock(vm); > + ret = libxlDomainStart(driver, vm, paused, recvfd); > + virObjectUnlock(vm); > + > + if (ret < 0 && !vm->persistent) > + virDomainObjListRemove(driver->domains, vm); > + > + cleanup: > + /* Remove all listen socks from event handler, and close them. */ > + for (i = 0; i < nsocks; i++) { > + virNetSocketUpdateIOCallback(socks[i], 0); > + virNetSocketRemoveIOCallback(socks[i]); > + virNetSocketClose(socks[i]); > + virObjectUnref(socks[i]); > I think you should set socks[i] = NULL and after the loop also set args->nsocks = 0, just to ensure nothing else can access these now free'd socks.

> +char * > +libxlDomainMigrationBegin(virConnectPtr conn, > + virDomainObjPtr vm, > + const char *xmlin) > +{ > + libxlDriverPrivatePtr driver = conn->privateData; > + libxlDriverConfigPtr cfg = libxlDriverConfigGet(driver); > + virDomainDefPtr tmpdef = NULL; > + virDomainDefPtr def; > + char *xml = NULL; > + > + if (libxlDomainObjBeginJob(driver, vm, LIBXL_JOB_MODIFY) < 0) > + goto cleanup; > + > + if (xmlin) { > + if (!(tmpdef = virDomainDefParseString(xmlin, cfg->caps, > + driver->xmlopt, > + 1 << VIR_DOMAIN_VIRT_XEN, > + VIR_DOMAIN_XML_INACTIVE))) > + goto endjob; > I think you want to call an equivalent of qemuDomainDefCheckABIStability here. THis ensures the user supplied XML only contains changes that are not going to impact guest machine ABI.