[libvirt] [PATCH] Support a new peer-to-peer migration mode & public API

Introduces several new public API options for migration - VIR_MIGRATE_PEER2PEER: With this flag the client only invokes the virDomainMigratePerform method, expecting the source host driver to do whatever is required to complete the entire migration process. - VIR_MIGRATE_TUNNELLED: With this flag the actual data for migration will be tunnelled over the libvirtd RPC channel. This requires that VIR_MIGRATE_PEER2PEER is also set. - virDomainMigrateToURI: This is variant of the existing virDomainMigrate method which can be used when the VIR_MIGRATE_PEER2PEER flag is set. The benefit of this method is that no virConnectPtr object is required for the destination host, only a destination URI. The URI for VIR_MIGRATE_TUNNELLED must be a valid libvirt URI. For non-tunnelled migration a hypervisor specific migration URI is used. * include/libvirt/libvirt.h, include/libvirt/libvirt.h.in: Add VIR_MIGRATE_PEER2PEER, VIR_MIGRATE_TUNNELLED and virDomainMigrateToURI. * src/driver.h: Remove feature flags * src/libvirt_internal.h: Add feature flags, and include new VIR_FEATURE_MIGRATE_P2P indicating support for the new VIR_MIGRATE_PEER2PEER mode. * src/libvirt.c: Implement support for VIR_MIGRATE_PEER2PEER and virDomainMigrateToURI APIs. * src/virsh.c: Add --p2p and --tunnelled args and use the new virDomainMigrateToURI method where possible. * src/xen_unified.c: Advertise support for P2P migration * src/xend_internal.c: Accept VIR_MIGRATE_PEER2PEER flag. * src/libvirt_public.syms: Export virDomainMigrateToURI method --- include/libvirt/libvirt.h | 8 ++- include/libvirt/libvirt.h.in | 8 ++- src/driver.h | 19 ---- src/internal.h | 2 + src/libvirt.c | 224 +++++++++++++++++++++++++++++++++++++----- src/libvirt_internal.h | 30 ++++++ src/libvirt_public.syms | 1 + src/virsh.c | 42 ++++++-- src/xen_unified.c | 7 +- src/xend_internal.c | 6 + 10 files changed, 286 insertions(+), 61 deletions(-) diff --git a/include/libvirt/libvirt.h b/include/libvirt/libvirt.h index 10ec04a..1745396 100644 --- a/include/libvirt/libvirt.h +++ b/include/libvirt/libvirt.h @@ -336,7 +336,9 @@ typedef virDomainInterfaceStatsStruct *virDomainInterfaceStatsPtr; /* Domain migration flags. */ typedef enum { - VIR_MIGRATE_LIVE = 1, /* live migration */ + VIR_MIGRATE_LIVE = (1 << 0), /* live migration */ + VIR_MIGRATE_PEER2PEER = (1 << 1), /* direct source -> dest host control channel */ + VIR_MIGRATE_TUNNELLED = (1 << 2), /* tunnel migration data over libvirtd connection */ } virDomainMigrateFlags; /* Domain migration. */ @@ -344,6 +346,10 @@ virDomainPtr virDomainMigrate (virDomainPtr domain, virConnectPtr dconn, unsigned long flags, const char *dname, const char *uri, unsigned long bandwidth); +int virDomainMigrateToURI (virDomainPtr domain, const char *duri, + unsigned long flags, const char *dname, + unsigned long bandwidth); + /** * VIR_NODEINFO_MAXCPUS: * @nodeinfo: virNodeInfo instance diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in index 012d30e..5e1a500 100644 --- a/include/libvirt/libvirt.h.in +++ b/include/libvirt/libvirt.h.in @@ -336,7 +336,9 @@ typedef virDomainInterfaceStatsStruct *virDomainInterfaceStatsPtr; /* Domain migration flags. */ typedef enum { - VIR_MIGRATE_LIVE = 1, /* live migration */ + VIR_MIGRATE_LIVE = (1 << 0), /* live migration */ + VIR_MIGRATE_PEER2PEER = (1 << 1), /* direct source -> dest host control channel */ + VIR_MIGRATE_TUNNELLED = (1 << 2), /* tunnel migration data over libvirtd connection */ } virDomainMigrateFlags; /* Domain migration. */ @@ -344,6 +346,10 @@ virDomainPtr virDomainMigrate (virDomainPtr domain, virConnectPtr dconn, unsigned long flags, const char *dname, const char *uri, unsigned long bandwidth); +int virDomainMigrateToURI (virDomainPtr domain, const char *duri, + unsigned long flags, const char *dname, + unsigned long bandwidth); + /** * VIR_NODEINFO_MAXCPUS: * @nodeinfo: virNodeInfo instance diff --git a/src/driver.h b/src/driver.h index 6a3dcc2..893e98b 100644 --- a/src/driver.h +++ b/src/driver.h @@ -44,25 +44,6 @@ typedef enum { VIR_DRV_OPEN_ERROR = -2, } virDrvOpenStatus; -/* Feature detection. This is a libvirt-private interface for determining - * what features are supported by the driver. - * - * The remote driver passes features through to the real driver at the - * remote end unmodified, except if you query a VIR_DRV_FEATURE_REMOTE* - * feature. - */ - /* Driver supports V1-style virDomainMigrate, ie. domainMigratePrepare/ - * domainMigratePerform/domainMigrateFinish. - */ -#define VIR_DRV_FEATURE_MIGRATION_V1 1 - - /* Driver is not local. */ -#define VIR_DRV_FEATURE_REMOTE 2 - - /* Driver supports V2-style virDomainMigrate, ie. domainMigratePrepare2/ - * domainMigratePerform/domainMigrateFinish2. - */ -#define VIR_DRV_FEATURE_MIGRATION_V2 3 /* Internal feature-detection macro. Don't call drv->supports_feature * directly, because it may be NULL, use this macro instead. diff --git a/src/internal.h b/src/internal.h index 8fa579c..bd1cfe6 100644 --- a/src/internal.h +++ b/src/internal.h @@ -24,6 +24,8 @@ #include "libvirt/libvirt.h" #include "libvirt/virterror.h" +#include "libvirt_internal.h" + /* On architectures which lack these limits, define them (ie. Cygwin). * Note that the libvirt code should be robust enough to handle the * case where actual value is longer than these limits (eg. by setting diff --git a/src/libvirt.c b/src/libvirt.c index 124e5db..481afec 100644 --- a/src/libvirt.c +++ b/src/libvirt.c @@ -35,7 +35,6 @@ #include "virterror_internal.h" #include "logging.h" #include "datatypes.h" -#include "libvirt_internal.h" #include "driver.h" #include "uuid.h" @@ -3059,6 +3058,38 @@ virDomainMigrateVersion2 (virDomainPtr domain, return ddomain; } + + /* + * This is sort of a migration v3 + * + * This performs a peer-2-peer migration where source host + * does all the communication with the destination host. + */ +static int +virDomainMigrateP2P (virDomainPtr domain, + unsigned long flags, + const char *dname, + const char *uri, + unsigned long bandwidth) +{ + if (!domain->conn->driver->domainMigratePerform) { + virLibConnError (domain->conn, VIR_ERR_NO_SUPPORT, __FUNCTION__); + return -1; + } + + /* Perform the migration. The driver isn't supposed to return + * until the migration is complete. + */ + return domain->conn->driver->domainMigratePerform(domain, + NULL, /* cookie */ + 0, /* cookielen */ + uri, + flags, + dname, + bandwidth); +} + + /** * virDomainMigrate: * @domain: a domain object @@ -3072,22 +3103,34 @@ virDomainMigrateVersion2 (virDomainPtr domain, * host given by dconn (a connection to the destination host). * * Flags may be one of more of the following: - * VIR_MIGRATE_LIVE Attempt a live migration. + * VIR_MIGRATE_LIVE Do not pause the VM during migration + * VIR_MIGRATE_PEER2PEER Direct connection between source & destination hosts + * VIR_MIGRATE_TUNNELLED Tunnel migration data over the libvirt RPC channel + * + * VIR_MIGRATE_TUNNELLED requires that VIR_MIGRATE_PEER2PEER be set. + * Applications using the VIR_MIGRATE_PEER2PEER flag will probably + * prefer to invoke virDomainMigrateToURI, avoiding the need to + * open connection to the destination host themselves. * * If a hypervisor supports renaming domains during migration, * then you may set the dname parameter to the new name (otherwise * it keeps the same name). If this is not supported by the * hypervisor, dname must be NULL or else you will get an error. * - * Since typically the two hypervisors connect directly to each - * other in order to perform the migration, you may need to specify - * a path from the source to the destination. This is the purpose - * of the uri parameter. If uri is NULL, then libvirt will try to - * find the best method. Uri may specify the hostname or IP address - * of the destination host as seen from the source. Or uri may be - * a URI giving transport, hostname, user, port, etc. in the usual - * form. Refer to driver documentation for the particular URIs - * supported. + * If the VIR_MIGRATE_TUNNELLED flag is set, the uri parameter + * must be a valid libvirt connection URI, by which the source + * libvirt driver can connect to the destination libvirt. If + * omitted, the dconn connection object will be queried for its + * current URI. + * + * If the VIR_MIGRATE_TUNNELED flag is NOT set, the URI parameter + * takes a hypervisor specific format. The hypervisor capabilities + * XML includes details of the support URI schemes. If omitted + * the dconn will be asked for a default URI. + * + * In either case it is typically only neccessary to specify a + * URI if the destination host has multiple interfaces and a + * specific interface is required to transmit migration data. * * The maximum bandwidth (in Mbps) that will be used to do migration * can be specified with the bandwidth parameter. If set to 0, @@ -3142,24 +3185,50 @@ virDomainMigrate (virDomainPtr domain, goto error; } - /* Check that migration is supported by both drivers. */ - if (VIR_DRV_SUPPORTS_FEATURE (domain->conn->driver, domain->conn, - VIR_DRV_FEATURE_MIGRATION_V1) && - VIR_DRV_SUPPORTS_FEATURE (dconn->driver, dconn, - VIR_DRV_FEATURE_MIGRATION_V1)) - ddomain = virDomainMigrateVersion1 (domain, dconn, flags, dname, uri, bandwidth); - else if (VIR_DRV_SUPPORTS_FEATURE (domain->conn->driver, domain->conn, - VIR_DRV_FEATURE_MIGRATION_V2) && - VIR_DRV_SUPPORTS_FEATURE (dconn->driver, dconn, - VIR_DRV_FEATURE_MIGRATION_V2)) - ddomain = virDomainMigrateVersion2 (domain, dconn, flags, dname, uri, bandwidth); - else { - virLibConnError (domain->conn, VIR_ERR_NO_SUPPORT, __FUNCTION__); - goto error; + if (flags & VIR_MIGRATE_PEER2PEER) { + if (VIR_DRV_SUPPORTS_FEATURE (domain->conn->driver, domain->conn, + VIR_DRV_FEATURE_MIGRATION_P2P)) { + char *duri = NULL; + if (!uri) + duri = virConnectGetURI(dconn); + + if (virDomainMigrateP2P (domain, flags, dname, uri ? uri : duri, bandwidth) < 0) { + VIR_FREE(duri); + goto error; + } + VIR_FREE(duri); + + ddomain = virDomainLookupByName (dconn, dname ? dname : domain->name); + } else { + virLibConnError (domain->conn, VIR_ERR_NO_SUPPORT, __FUNCTION__); + goto error; + } + } else { + if (flags & VIR_MIGRATE_TUNNELLED) { + virLibConnError(domain->conn, VIR_ERR_OPERATION_INVALID, + _("cannot perform tunnelled migration without using peer2peer flag")); + goto error; + } + + /* Check that migration is supported by both drivers. */ + if (VIR_DRV_SUPPORTS_FEATURE (domain->conn->driver, domain->conn, + VIR_DRV_FEATURE_MIGRATION_V1) && + VIR_DRV_SUPPORTS_FEATURE (dconn->driver, dconn, + VIR_DRV_FEATURE_MIGRATION_V1)) + ddomain = virDomainMigrateVersion1 (domain, dconn, flags, dname, uri, bandwidth); + else if (VIR_DRV_SUPPORTS_FEATURE (domain->conn->driver, domain->conn, + VIR_DRV_FEATURE_MIGRATION_V2) && + VIR_DRV_SUPPORTS_FEATURE (dconn->driver, dconn, + VIR_DRV_FEATURE_MIGRATION_V2)) + ddomain = virDomainMigrateVersion2 (domain, dconn, flags, dname, uri, bandwidth); + else { + virLibConnError (domain->conn, VIR_ERR_NO_SUPPORT, __FUNCTION__); + goto error; + } } - if (ddomain == NULL) - goto error; + if (ddomain == NULL) + goto error; return ddomain; @@ -3169,6 +3238,107 @@ error: return NULL; } + +/** + * virDomainMigrateToURI: + * @domain: a domain object + * @duri: libvirt connection URI for destination host + * @flags: flags + * @dname: (optional) rename domain to this at destination + * @bandwidth: (optional) specify migration bandwidth limit in Mbps + * + * Migrate the domain object from its current host to the destination + * host given by duri. The duri is a libvirt connection URI that the + * source host will use to talk to the destination. + * + * The difference from the regular virDomainMigrate method is + * that the calling application does not need a direct connection + * to the destination host. The source libvirt driver makes a + * direct peer-to-peer connection to the destination libvirtd + * without the client appliction being involved. + * + * The VIR_MIGRATE_PEER2PEER flag is mandatory for this method. + * If an application wishes to run without this flag, then it + * may use the alternative virDomainMigrate method which requires + * an virConnectPtr object for the destination host + * + * The following additional flags may also be set + * VIR_MIGRATE_LIVE Do not pause the VM during migration + * VIR_MIGRATE_TUNNELLED Tunnel migration data over the libvirt RPC channel + * + * If a hypervisor supports renaming domains during migration, + * then you may set the dname parameter to the new name (otherwise + * it keeps the same name). If this is not supported by the + * hypervisor, dname must be NULL or else you will get an error. + * + * The maximum bandwidth (in Mbps) that will be used to do migration + * can be specified with the bandwidth parameter. If set to 0, + * libvirt will choose a suitable default. Some hypervisors do + * not support this feature and will return an error if bandwidth + * is not 0. + * + * To see which features are supported by the current hypervisor, + * see virConnectGetCapabilities, /capabilities/host/migration_features. + * + * There are many limitations on migration imposed by the underlying + * technology - for example it may not be possible to migrate between + * different processors even with the same architecture, or between + * different types of hypervisor. + * + * Returns 0 if the migration succeeded, -1 upon error. + */ +int +virDomainMigrateToURI (virDomainPtr domain, + const char *duri, + unsigned long flags, + const char *dname, + unsigned long bandwidth) +{ + DEBUG("domain=%p, duri=%p, flags=%lu, dname=%s, bandwidth=%lu", + domain, NULLSTR(duri), flags, NULLSTR(dname), bandwidth); + + virResetLastError(); + + /* First checkout the source */ + if (!VIR_IS_CONNECTED_DOMAIN (domain)) { + virLibDomainError(NULL, VIR_ERR_INVALID_DOMAIN, __FUNCTION__); + return -1; + } + if (domain->conn->flags & VIR_CONNECT_RO) { + virLibDomainError(domain, VIR_ERR_OPERATION_DENIED, __FUNCTION__); + goto error; + } + + if (duri == NULL) { + virLibConnError (domain->conn, VIR_ERR_INVALID_ARG, __FUNCTION__); + goto error; + } + + /* XXX perhaps we should just automatically set this flag instead ? */ + if (flags & VIR_MIGRATE_PEER2PEER) { + if (VIR_DRV_SUPPORTS_FEATURE (domain->conn->driver, domain->conn, + VIR_DRV_FEATURE_MIGRATION_P2P)) { + if (virDomainMigrateP2P (domain, flags, dname, duri, bandwidth) < 0) + goto error; + } else { + virLibConnError (domain->conn, VIR_ERR_NO_SUPPORT, __FUNCTION__); + goto error; + } + } else { + virLibConnError (domain->conn, VIR_ERR_OPERATION_INVALID, + _("cannot migrate to a destination URI without peer2peer flag")); + goto error; + } + + return 0; + +error: + /* Copy to connection error object for back compatability */ + virSetConnError(domain->conn); + return -1; +} + + /* * Not for public use. This function is part of the internal * implementation of migration in the remote case. diff --git a/src/libvirt_internal.h b/src/libvirt_internal.h index 5913798..5f1a7fe 100644 --- a/src/libvirt_internal.h +++ b/src/libvirt_internal.h @@ -17,6 +17,7 @@ * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * + * NB This file is (secret) ABI sensitive. Append only */ #ifndef __LIBVIRT_H_ @@ -31,6 +32,35 @@ int virStateReload(void); int virStateActive(void); #endif +/* Feature detection. This is a libvirt-private interface for determining + * what features are supported by the driver. + * + * The remote driver passes features through to the real driver at the + * remote end unmodified, except if you query a VIR_DRV_FEATURE_REMOTE* + * feature. + * + */ +enum { + /* Driver supports V1-style virDomainMigrate, ie. domainMigratePrepare/ + * domainMigratePerform/domainMigrateFinish. + */ + VIR_DRV_FEATURE_MIGRATION_V1 = 1, + + /* Driver is not local. */ + VIR_DRV_FEATURE_REMOTE = 2, + + /* Driver supports V2-style virDomainMigrate, ie. domainMigratePrepare2/ + * domainMigratePerform/domainMigrateFinish2. + */ + VIR_DRV_FEATURE_MIGRATION_V2 = 3, + + /* Driver supports peer-2-peer virDomainMigrate ie soruce host + * do all the prepare/perform/finish steps directly + */ + VIR_DRV_FEATURE_MIGRATION_P2P = 4, +}; + + int virDrvSupportsFeature (virConnectPtr conn, int feature); int virDomainMigratePrepare (virConnectPtr dconn, diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms index 888ea26..757e54c 100644 --- a/src/libvirt_public.syms +++ b/src/libvirt_public.syms @@ -323,6 +323,7 @@ LIBVIRT_0.7.1 { virStreamFinish; virStreamAbort; virStreamFree; + virDomainMigrateToURI; } LIBVIRT_0.7.0; # .... define new API here using predicted next version number .... diff --git a/src/virsh.c b/src/virsh.c index 4825f1c..cabbd3d 100644 --- a/src/virsh.c +++ b/src/virsh.c @@ -2463,6 +2463,8 @@ static const vshCmdInfo info_migrate[] = { static const vshCmdOptDef opts_migrate[] = { {"live", VSH_OT_BOOL, 0, gettext_noop("live migration")}, + {"p2p", VSH_OT_BOOL, 0, gettext_noop("peer-2-peer migration")}, + {"tunnelled", VSH_OT_BOOL, 0, gettext_noop("tunnelled migration")}, {"domain", VSH_OT_DATA, VSH_OFLAG_REQ, gettext_noop("domain name, id or uuid")}, {"desturi", VSH_OT_DATA, VSH_OFLAG_REQ, gettext_noop("connection URI of the destination host")}, {"migrateuri", VSH_OT_DATA, 0, gettext_noop("migration URI, usually can be omitted")}, @@ -2478,8 +2480,6 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd) const char *migrateuri; const char *dname; int flags = 0, found, ret = FALSE; - virConnectPtr dconn = NULL; - virDomainPtr ddom = NULL; if (!vshConnectionUsability (ctl, ctl->conn, TRUE)) return FALSE; @@ -2500,20 +2500,40 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd) if (vshCommandOptBool (cmd, "live")) flags |= VIR_MIGRATE_LIVE; - /* Temporarily connect to the destination host. */ - dconn = virConnectOpenAuth (desturi, virConnectAuthPtrDefault, 0); - if (!dconn) goto done; + if (vshCommandOptBool (cmd, "p2p")) + flags |= VIR_MIGRATE_PEER2PEER; - /* Migrate. */ - ddom = virDomainMigrate (dom, dconn, flags, dname, migrateuri, 0); - if (!ddom) goto done; + if (vshCommandOptBool (cmd, "tunnelled")) + flags |= VIR_MIGRATE_TUNNELLED; - ret = TRUE; + if ((flags & VIR_MIGRATE_PEER2PEER)) { + /* For peer2peer migration we only expect one URI, a libvirt URI */ + + if (migrateuri != NULL) { + vshError(ctl, FALSE, "%s", _("migrate: Unexpected migrateuri for peer2peer migration")); + goto done; + } + + if (virDomainMigrateToURI (dom, desturi, flags, dname, 0) == 0) + ret = TRUE; + } else { + /* For regular live migration, connect to the destination host directly. */ + virConnectPtr dconn = NULL; + virDomainPtr ddom = NULL; + + dconn = virConnectOpenAuth (desturi, virConnectAuthPtrDefault, 0); + if (!dconn) goto done; + + ddom = virDomainMigrate (dom, dconn, flags, dname, migrateuri, 0); + if (ddom) { + virDomainFree(ddom); + ret = TRUE; + } + virConnectClose (dconn); + } done: if (dom) virDomainFree (dom); - if (ddom) virDomainFree (ddom); - if (dconn) virConnectClose (dconn); return ret; } diff --git a/src/xen_unified.c b/src/xen_unified.c index dfa9ca5..954b187 100644 --- a/src/xen_unified.c +++ b/src/xen_unified.c @@ -455,8 +455,11 @@ static int xenUnifiedSupportsFeature (virConnectPtr conn ATTRIBUTE_UNUSED, int feature) { switch (feature) { - case VIR_DRV_FEATURE_MIGRATION_V1: return 1; - default: return 0; + case VIR_DRV_FEATURE_MIGRATION_V1: + case VIR_DRV_FEATURE_MIGRATION_P2P: + return 1; + default: + return 0; } } diff --git a/src/xend_internal.c b/src/xend_internal.c index 7f55116..da5c039 100644 --- a/src/xend_internal.c +++ b/src/xend_internal.c @@ -4400,6 +4400,12 @@ xenDaemonDomainMigratePerform (virDomainPtr domain, strcpy (live, "1"); flags &= ~VIR_MIGRATE_LIVE; } + /* Trivially support this in Xen, since XenD on dest is always + * ready to accept incoming migration */ + if ((flags & VIR_MIGRATE_PEER2PEER)) { + flags &= ~VIR_MIGRATE_PEER2PEER; + } + /* XXX we could easily do tunnelled migration too if we want to */ if (flags != 0) { virXendError (conn, VIR_ERR_NO_SUPPORT, "%s", _("xenDaemonDomainMigrate: unsupported flag")); -- 1.6.2.5