Re: [libvirt] [PATCH 1/6] vz: add migration backbone code

From: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; This patch makes basic vz migration possible. For example by virsh: virsh -c vz:///system migrate $NAME vz+ssh://$DST/system Vz migration is implemented thru interface for managed migrations for drivers although it looks like a candadate for direct migration as all work is done by vz sdk. The reason is that vz sdk lacks rich remote authentication capabilities of libvirt and if we choose to implement direct migration we have to reimplement auth means of libvirt. This brings the requirement that destination side should have running libvirt daemon. This is not the problem as vz is moving in the direction of tight integration with libvirt. Another issue of this choice is that if the managment migration fails on 'finish' step driver is supposed to resume on source. This is not compatible with vz sdk migration but this can be overcome without loosing a constistency, see comments in code. Technically we have a libvirt connection to destination in managed migration scheme and we use this connection to obtain a session_uuid (which acts as authZ token) for vz migration. This uuid is passed from destination through cookie on 'prepare' step. A few words on vz migration uri. I'd probably use just 'hostname:port' uris as we don't have different migration schemes in vz but scheme part is mandatory, so 'tcp' is used. Looks like good name. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- src/vz/vz_driver.c | 250 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/vz/vz_sdk.c | 79 ++++++++++++++-- src/vz/vz_sdk.h | 2 + src/vz/vz_utils.h | 1 + 4 files changed, 322 insertions(+), 10 deletions(-) diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c index 9f0c52f..e003646 100644 --- a/src/vz/vz_driver.c +++ b/src/vz/vz_driver.c @@ -1343,6 +1343,250 @@ vzDomainMemoryStats(virDomainPtr domain, return ret; } +static int +vzConnectSupportsFeature(virConnectPtr conn ATTRIBUTE_UNUSED, int feature) +{ + switch (feature) { + case VIR_DRV_FEATURE_MIGRATION_PARAMS: + return 1; + default: + return 0; + } +} + +#define VZ_MIGRATION_PARAMETERS NULL + +static char * +vzDomainMigrateBegin3Params(virDomainPtr domain, + virTypedParameterPtr params, + int nparams, + char **cookieout ATTRIBUTE_UNUSED, + int *cookieoutlen ATTRIBUTE_UNUSED, + unsigned int fflags ATTRIBUTE_UNUSED) +{ + virDomainObjPtr dom = NULL; + char *xml = NULL; + + if (virTypedParamsValidate(params, nparams, VZ_MIGRATION_PARAMETERS) < 0) + goto cleanup; + + if (!(dom = vzDomObjFromDomain(domain))) + goto cleanup; + + xml = virDomainDefFormat(dom->def, VIR_DOMAIN_DEF_FORMAT_SECURE); + + cleanup: + if (dom) + virObjectUnlock(dom); + + return xml; +} + +/* return 'hostname' */ +static char * +vzCreateMigrateUri(void) +{ + char *hostname = NULL; + char *out = NULL; + virURI uri = {}; + + if ((hostname = virGetHostname()) == NULL) + goto cleanup; + + if (STRPREFIX(hostname, "localhost")) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("hostname on destination resolved to localhost," + " but migration requires an FQDN")); + goto cleanup; + } + + /* to set const string to non-const */ + if (VIR_STRDUP(uri.scheme, "tcp") < 0) + goto cleanup; + uri.server = hostname; + out = virURIFormat(&uri); + + cleanup: + VIR_FREE(hostname); + VIR_FREE(uri.scheme); + return out; +} + +static int +vzDomainMigratePrepare3Params(virConnectPtr dconn, + virTypedParameterPtr params ATTRIBUTE_UNUSED, + int nparams ATTRIBUTE_UNUSED, + const char *cookiein ATTRIBUTE_UNUSED, + int cookieinlen ATTRIBUTE_UNUSED, + char **cookieout, + int *cookieoutlen, + char **uri_out, + unsigned int fflags ATTRIBUTE_UNUSED) +{ + vzConnPtr privconn = dconn->privateData; + int ret = -1; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + *cookieout = NULL; + *uri_out = NULL; + + virUUIDFormat(privconn->session_uuid, uuidstr); + if (VIR_STRDUP(*cookieout, uuidstr) < 0) + goto cleanup; + *cookieoutlen = strlen(*cookieout) + 1; + + if (!(*uri_out = vzCreateMigrateUri())) + goto cleanup; + + ret = 0; + + cleanup: + if (ret != 0) { + VIR_FREE(*cookieout); + VIR_FREE(*uri_out); + *cookieoutlen = 0; + } + + return ret; +} + +static int +vzDomainMigratePerform3Params(virDomainPtr domain, + const char *dconnuri ATTRIBUTE_UNUSED, + virTypedParameterPtr params, + int nparams, + const char *cookiein, + int cookieinlen ATTRIBUTE_UNUSED, + char **cookieout, + int *cookieoutlen, + unsigned int fflags ATTRIBUTE_UNUSED) +{ + int ret = -1; + virDomainObjPtr dom = NULL; + const char *uri = NULL; + unsigned char session_uuid[VIR_UUID_BUFLEN]; + char uuidstr[VIR_UUID_STRING_BUFLEN]; + + *cookieout = NULL; + + if (virTypedParamsGetString(params, nparams, + VIR_MIGRATE_PARAM_URI, + &uri) < 0) + goto cleanup; + + if (!(dom = vzDomObjFromDomain(domain))) + goto cleanup; + + if (!uri) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("migration URI should be provided")); + goto cleanup; + } + + if (virUUIDParse(cookiein, session_uuid) < 0) + goto cleanup; + + if (prlsdkMigrate(dom, uri, session_uuid) < 0) + goto cleanup; + + virUUIDFormat(domain->uuid, uuidstr); + if (VIR_STRDUP(*cookieout, uuidstr) < 0) + goto cleanup; + *cookieoutlen = strlen(*cookieout) + 1; + + ret = 0; + + cleanup: + if (dom) + virObjectUnlock(dom); + if (ret != 0) { + VIR_FREE(*cookieout); + *cookieoutlen = 0; + } + + return ret; +} + +/* if we return NULL from this function we are supposed + to cleanup destination side, but we can't do it + because 'perform' step is finished and migration is actually + completed by dispatcher. Unfortunately in OOM situation we + have to return NULL. As a result high level migration + function return NULL which is supposed to be + treated as migration error while migration is + actually successful. This should not be a problem + as we are in consistent state. For example later + attempts to list source and destination domains + will reveal actual situation. */ + +static virDomainPtr +vzDomainMigrateFinish3Params(virConnectPtr dconn, + virTypedParameterPtr params ATTRIBUTE_UNUSED, + int nparams ATTRIBUTE_UNUSED, + const char *cookiein, + int cookieinlen ATTRIBUTE_UNUSED, + char **cookieout ATTRIBUTE_UNUSED, + int *cookieoutlen ATTRIBUTE_UNUSED, + unsigned int fflags ATTRIBUTE_UNUSED, + int cancelled) +{ + vzConnPtr privconn = dconn->privateData; + virDomainObjPtr dom = NULL; + virDomainPtr domain = NULL; + unsigned char domain_uuid[VIR_UUID_BUFLEN]; + + /* we have nothing to cleanup, whole job is done by PCS dispatcher */ + if (cancelled) + return NULL; + + if (virUUIDParse(cookiein, domain_uuid) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Could not parse UUID from string '%s'"), + cookiein); + goto cleanup; + } + + if (!(dom = prlsdkAddDomain(privconn, domain_uuid))) + goto cleanup; + + domain = virGetDomain(dconn, dom->def->name, dom->def->uuid); + if (domain) + domain->id = dom->def->id; + + cleanup: + if (!domain) + VIR_WARN("Can't provide domain with uuid '%s' after successfull migration.", cookiein); + virDomainObjEndAPI(&dom); + return domain; +} + +/* This is executed only if 'perform' step is successfull that + is migration is completed by PCS dispatcher. Thus we should + ignore 'canceled' parameter and always kill source domain. */ +static int +vzDomainMigrateConfirm3Params(virDomainPtr domain, + virTypedParameterPtr params ATTRIBUTE_UNUSED, + int nparams ATTRIBUTE_UNUSED, + const char *cookiein ATTRIBUTE_UNUSED, + int cookieinlen ATTRIBUTE_UNUSED, + unsigned int fflags ATTRIBUTE_UNUSED, + int cancelled ATTRIBUTE_UNUSED) +{ + vzConnPtr privconn = domain->conn->privateData; + virDomainObjPtr dom = NULL; + + if (!(dom = vzDomObjFromDomain(domain))) + goto cleanup; + + virDomainObjListRemove(privconn->domains, dom); + + cleanup: + if (dom) + virObjectUnlock(dom); + + return 0; +} + static virHypervisorDriver vzDriver = { .name = "vz", .connectOpen = vzConnectOpen, /* 0.10.0 */ @@ -1396,6 +1640,12 @@ static virHypervisorDriver vzDriver = { .domainBlockStatsFlags = vzDomainBlockStatsFlags, /* 1.2.17 */ .domainInterfaceStats = vzDomainInterfaceStats, /* 1.2.17 */ .domainMemoryStats = vzDomainMemoryStats, /* 1.2.17 */ + .connectSupportsFeature = vzConnectSupportsFeature, /* 1.2.18 */ + .domainMigrateBegin3Params = vzDomainMigrateBegin3Params, /* 1.2.18 */ + .domainMigratePrepare3Params = vzDomainMigratePrepare3Params, /* 1.2.18 */ + .domainMigratePerform3Params = vzDomainMigratePerform3Params, /* 1.2.18 */ + .domainMigrateFinish3Params = vzDomainMigrateFinish3Params, /* 1.2.18 */ + .domainMigrateConfirm3Params = vzDomainMigrateConfirm3Params, /* 1.2.18 */ }; static virConnectDriver vzConnectDriver = { diff --git a/src/vz/vz_sdk.c b/src/vz/vz_sdk.c index d1bc312..908bfc1 100644 --- a/src/vz/vz_sdk.c +++ b/src/vz/vz_sdk.c @@ -37,6 +37,9 @@ #define VIR_FROM_THIS VIR_FROM_PARALLELS #define JOB_INFINIT_WAIT_TIMEOUT UINT_MAX +static int +prlsdkUUIDParse(const char *uuidstr, unsigned char *uuid); + VIR_LOG_INIT("parallels.sdk"); /* @@ -228,24 +231,40 @@ prlsdkDeinit(void) int prlsdkConnect(vzConnPtr privconn) { - PRL_RESULT ret; + int ret = -1; + PRL_RESULT pret; PRL_HANDLE job = PRL_INVALID_HANDLE; + PRL_HANDLE result = PRL_INVALID_HANDLE; + PRL_HANDLE response = PRL_INVALID_HANDLE; + char session_uuid[VIR_UUID_STRING_BUFLEN + 2]; + PRL_UINT32 buflen = ARRAY_CARDINALITY(session_uuid); - ret = PrlSrv_Create(&privconn->server); - if (PRL_FAILED(ret)) { - logPrlError(ret); - return -1; - } + pret = PrlSrv_Create(&privconn->server); + prlsdkCheckRetGoto(pret, cleanup); job = PrlSrv_LoginLocalEx(privconn->server, NULL, 0, PSL_HIGH_SECURITY, PACF_NON_INTERACTIVE_MODE); + if (PRL_FAILED(getJobResult(job, &result))) + goto cleanup; + + pret = PrlResult_GetParam(result, &response); + prlsdkCheckRetGoto(pret, cleanup); + + pret = PrlLoginResponse_GetSessionUuid(response, session_uuid, &buflen); + prlsdkCheckRetGoto(pret, cleanup); + + if (prlsdkUUIDParse(session_uuid, privconn->session_uuid) < 0) + goto cleanup; + + ret = 0; - if (waitJob(job)) { + cleanup: + if (ret < 0) PrlHandle_Free(privconn->server); - return -1; - } + PrlHandle_Free(result); + PrlHandle_Free(response); - return 0; + return ret; } void @@ -4035,3 +4054,43 @@ prlsdkGetMemoryStats(virDomainObjPtr dom, return ret; } + +/* high security is default choice for 2 reasons: + 1. as this is the highest set security we can't get + reject from server with high security settings + 2. this is on par with security level of driver + connection to dispatcher */ + +#define PRLSDK_MIGRATION_FLAGS (PSL_HIGH_SECURITY) + +int prlsdkMigrate(virDomainObjPtr dom, const char* uri_str, + const unsigned char *session_uuid) +{ + int ret = -1; + vzDomObjPtr privdom = dom->privateData; + virURIPtr uri = NULL; + PRL_HANDLE job = PRL_INVALID_HANDLE; + char uuidstr[VIR_UUID_STRING_BUFLEN + 2]; + + uri = virURIParse(uri_str); + /* no special error logs as uri should be checked on prepare step */ + if (uri == NULL) + goto cleanup; + + prlsdkUUIDFormat(session_uuid, uuidstr); + job = PrlVm_MigrateEx(privdom->sdkdom, uri->server, uri->port, uuidstr, + "", /* use default dir for migrated instance bundle */ + PRLSDK_MIGRATION_FLAGS, + 0, /* reserved flags */ + PRL_TRUE /* don't ask for confirmations */ + ); + + if (PRL_FAILED(waitJob(job))) + goto cleanup; + + ret = 0; + + cleanup: + virURIFree(uri); + return ret; +} diff --git a/src/vz/vz_sdk.h b/src/vz/vz_sdk.h index ebe4591..1a90eca 100644 --- a/src/vz/vz_sdk.h +++ b/src/vz/vz_sdk.h @@ -76,3 +76,5 @@ int prlsdkGetVcpuStats(virDomainObjPtr dom, int idx, unsigned long long *time); int prlsdkGetMemoryStats(virDomainObjPtr dom, virDomainMemoryStatPtr stats, unsigned int nr_stats); +int +prlsdkMigrate(virDomainObjPtr dom, const char* uri_str, const char unsigned *session_uuid); diff --git a/src/vz/vz_utils.h b/src/vz/vz_utils.h index db09647..a779b03 100644 --- a/src/vz/vz_utils.h +++ b/src/vz/vz_utils.h @@ -62,6 +62,7 @@ struct _vzConn { virDomainObjListPtr domains; PRL_HANDLE server; + unsigned char session_uuid[VIR_UUID_BUFLEN]; virStoragePoolObjList pools; virNetworkObjListPtr networks; virCapsPtr caps;