From: WangYufei <james.wangyufei(a)huawei.com&gt; https://bugzilla.redhat.com/show_bug.cgi?id=1019053 When we migrate vms concurrently, there's a chance that libvirtd on destination assigns the same port for different migrations, which will lead to migration failure during prepare phase on destination. So we use virPortAllocator here to solve the problem. Signed-off-by: WangYufei <james.wangyufei(a)huawei.com&gt; Signed-off-by: Jiri Denemark <jdenemar(a)redhat.com&gt; --- src/qemu/qemu_command.h | 3 +++ src/qemu/qemu_conf.h | 6 +++--- src/qemu/qemu_domain.h | 1 + src/qemu/qemu_driver.c | 6 ++++++ src/qemu/qemu_migration.c | 39 +++++++++++++++++++++++++-------------- 5 files changed, 38 insertions(+), 17 deletions(-) diff --git a/src/qemu/qemu_command.h b/src/qemu/qemu_command.h index 2e2acfb..3277ba4 100644 --- a/src/qemu/qemu_command.h +++ b/src/qemu/qemu_command.h @@ -51,6 +51,9 @@ # define QEMU_WEBSOCKET_PORT_MIN 5700 # define QEMU_WEBSOCKET_PORT_MAX 65535 +# define QEMU_MIGRATION_PORT_MIN 49152 +# define QEMU_MIGRATION_PORT_MAX 49215 + typedef struct _qemuBuildCommandLineCallbacks qemuBuildCommandLineCallbacks; typedef qemuBuildCommandLineCallbacks *qemuBuildCommandLineCallbacksPtr; struct _qemuBuildCommandLineCallbacks { diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index ea3c691..863d215 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -225,6 +225,9 @@ struct _virQEMUDriver { /* Immutable pointer, self-locking APIs */ virPortAllocatorPtr webSocketPorts; + /* Immutable pointer, self-locking APIs */ + virPortAllocatorPtr migrationPorts; + /* Immutable pointer, lockless APIs*/ virSysinfoDefPtr hostsysinfo; @@ -246,9 +249,6 @@ struct _qemuDomainCmdlineDef { char **env_value; }; -/* Port numbers used for KVM migration. */ -# define QEMUD_MIGRATION_FIRST_PORT 49152 -# define QEMUD_MIGRATION_NUM_PORTS 64 void qemuDomainCmdlineDefFree(qemuDomainCmdlineDefPtr def); diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 5d9fab9..2f9d20f 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -160,6 +160,7 @@ struct _qemuDomainObjPrivate { unsigned long migMaxBandwidth; char *origname; int nbdPort; /* Port used for migration with NBD */ + int migrationPort; virChrdevsPtr devs; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 746da26..895681b 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -687,6 +687,11 @@ qemuStateInitialize(bool privileged, cfg->webSocketPortMax)) == NULL) goto error; + if ((qemu_driver->migrationPorts = + virPortAllocatorNew(QEMU_MIGRATION_PORT_MIN, + QEMU_MIGRATION_PORT_MAX)) == NULL) + goto error; + if (qemuSecurityInit(qemu_driver) < 0) goto error; @@ -993,6 +998,7 @@ qemuStateCleanup(void) { virObjectUnref(qemu_driver->domains); virObjectUnref(qemu_driver->remotePorts); virObjectUnref(qemu_driver->webSocketPorts); + virObjectUnref(qemu_driver->migrationPorts); virObjectUnref(qemu_driver->xmlopt); diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index 38edadb..a77aeb7 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -2141,6 +2141,9 @@ qemuMigrationPrepareCleanup(virQEMUDriverPtr driver, qemuDomainJobTypeToString(priv->job.active), qemuDomainAsyncJobTypeToString(priv->job.asyncJob)); + virPortAllocatorRelease(driver->migrationPorts, priv->migrationPort); + priv->migrationPort = 0; + if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_IN)) return; qemuDomainObjDiscardAsyncJob(driver, vm); @@ -2156,7 +2159,8 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, virDomainDefPtr *def, const char *origname, virStreamPtr st, - unsigned int port, + unsigned short port, + bool autoPort, const char *listenAddress, unsigned long flags) { @@ -2436,6 +2440,8 @@ done: goto cleanup; } + if (autoPort) + priv->migrationPort = port; ret = 0; cleanup: @@ -2503,7 +2509,7 @@ qemuMigrationPrepareTunnel(virQEMUDriverPtr driver, ret = qemuMigrationPrepareAny(driver, dconn, cookiein, cookieinlen, cookieout, cookieoutlen, def, origname, - st, 0, NULL, flags); + st, 0, false, NULL, flags); return ret; } @@ -2522,8 +2528,8 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, const char *listenAddress, unsigned long flags) { - static int port = 0; - int this_port; + unsigned short port = 0; + bool autoPort = true; char *hostname = NULL; const char *p; char *uri_str = NULL; @@ -2550,8 +2556,8 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, * to be a correct hostname which refers to the target machine). */ if (uri_in == NULL) { - this_port = QEMUD_MIGRATION_FIRST_PORT + port++; - if (port == QEMUD_MIGRATION_NUM_PORTS) port = 0; + if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) + goto cleanup; /* Get hostname */ if ((hostname = virGetHostname()) == NULL) @@ -2570,7 +2576,7 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, * new targets accept both syntaxes though. */ /* Caller frees */ - if (virAsprintf(uri_out, "tcp:%s:%d", hostname, this_port) < 0) + if (virAsprintf(uri_out, "tcp:%s:%d", hostname, port) < 0) goto cleanup; } else { /* Check the URI starts with "tcp:". We will escape the @@ -2607,16 +2613,16 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, if (uri->port == 0) { /* Generate a port */ - this_port = QEMUD_MIGRATION_FIRST_PORT + port++; - if (port == QEMUD_MIGRATION_NUM_PORTS) - port = 0; + if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) + goto cleanup; /* Caller frees */ - if (virAsprintf(uri_out, "%s:%d", uri_in, this_port) < 0) + if (virAsprintf(uri_out, "%s:%d", uri_in, port) < 0) goto cleanup; } else { - this_port = uri->port; + port = uri->port; + autoPort = false; } } @@ -2625,12 +2631,15 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, ret = qemuMigrationPrepareAny(driver, dconn, cookiein, cookieinlen, cookieout, cookieoutlen, def, origname, - NULL, this_port, listenAddress, flags); + NULL, port, autoPort, listenAddress, flags); cleanup: virURIFree(uri); VIR_FREE(hostname); - if (ret != 0) + if (ret != 0) { VIR_FREE(*uri_out); + if (autoPort) + virPortAllocatorRelease(driver->migrationPorts, port); + } return ret; } @@ -4410,6 +4419,8 @@ qemuMigrationFinish(virQEMUDriverPtr driver, } qemuMigrationStopNBDServer(driver, vm, mig); + virPortAllocatorRelease(driver->migrationPorts, priv->migrationPort); + priv->migrationPort = 0; if (flags & VIR_MIGRATE_PERSIST_DEST) { virDomainDefPtr vmdef; -- 1.8.3.2