Current libvirt + qemu pair lacks secure migrations in case of VMs with non-shared disks. The only option to migrate securely natively is to use tunneled mode and some kind of secure destination URI. But tunelled mode does not support non-shared disks. The other way to make migration secure is to organize a tunnel by external means. This is possible in case of shared disks migration thru use of proper combination of destination URI, migration URI and VIR_MIGRATE_PARAM_LISTEN_ADDRESS migration param. But again this is not possible in case of non shared disks migration as we have no option to control target nbd server port. But fixing this much more simplier that supporting non-shared disks in tunneled mode. So this patch adds option to set target ndb port. Finally all qemu migration connections will be secured AFAIK but even in this case this patch could be convinient if one wants all migration traffic be put in a single connection. Signed-off-by: Nikolay Shirokovskiy <nshirokovskiy(a)virtuozzo.com&gt; --- Difference from v1: 1. fix indentation 2. fix uninitialized variable case in qemuDomainMigratePerform3Params 3. remove check that nbd port that passed from target back to source on preparation step in cookie is requested one. First the check was malformed. Second we don't need it anyway. In case target does not support setting nbd port we get error ealier when checking variable parameters list. After this check is removed we don't need to pass port parameter here and there on perform step except for the tunnel case when we convert all parameters back to parameter list. include/libvirt/libvirt-domain.h | 10 +++++ src/qemu/qemu_driver.c | 25 ++++++++---- src/qemu/qemu_migration.c | 85 ++++++++++++++++++++++++++++------------ src/qemu/qemu_migration.h | 3 ++ tools/virsh-domain.c | 12 ++++++ tools/virsh.pod | 5 ++- 6 files changed, 106 insertions(+), 34 deletions(-) diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h index 65f1618..aa380f5 100644 --- a/include/libvirt/libvirt-domain.h +++ b/include/libvirt/libvirt-domain.h @@ -757,6 +757,16 @@ typedef enum { */ # define VIR_MIGRATE_PARAM_MIGRATE_DISKS "migrate_disks" +/** + * VIR_MIGRATE_PARAM_NBD_PORT: + * + * virDomainMigrate* params field: port that destination nbd server should use + * for incoming disks migration. Type is VIR_TYPED_PARAM_INT. If set to 0 or + * omitted, libvirt will choose a suitable default. At the moment this is only + * supported by the QEMU driver. + */ +# define VIR_MIGRATE_PARAM_NBD_PORT "nbd_port" + /* Domain migration. */ virDomainPtr virDomainMigrate (virDomainPtr domain, virConnectPtr dconn, unsigned long flags, const char *dname, diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 2bbc724..6230087 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -12005,7 +12005,7 @@ qemuDomainMigratePrepare2(virConnectPtr dconn, ret = qemuMigrationPrepareDirect(driver, dconn, NULL, 0, NULL, NULL, /* No cookies */ uri_in, uri_out, - &def, origname, NULL, 0, NULL, flags); + &def, origname, NULL, 0, NULL, 0, flags); cleanup: VIR_FREE(origname); @@ -12058,7 +12058,7 @@ qemuDomainMigratePerform(virDomainPtr dom, * Consume any cookie we were able to decode though */ ret = qemuMigrationPerform(driver, dom->conn, vm, - NULL, dconnuri, uri, NULL, NULL, 0, NULL, + NULL, dconnuri, uri, NULL, NULL, 0, NULL, 0, cookie, cookielen, NULL, NULL, /* No output cookies in v2 */ flags, dname, resource, false); @@ -12231,7 +12231,7 @@ qemuDomainMigratePrepare3(virConnectPtr dconn, cookiein, cookieinlen, cookieout, cookieoutlen, uri_in, uri_out, - &def, origname, NULL, 0, NULL, flags); + &def, origname, NULL, 0, NULL, 0, flags); cleanup: VIR_FREE(origname); @@ -12257,6 +12257,7 @@ qemuDomainMigratePrepare3Params(virConnectPtr dconn, const char *dname = NULL; const char *uri_in = NULL; const char *listenAddress = cfg->migrationAddress; + int nbdPort = 0; int nmigrate_disks; const char **migrate_disks = NULL; char *origname = NULL; @@ -12277,7 +12278,10 @@ qemuDomainMigratePrepare3Params(virConnectPtr dconn, &uri_in) < 0 || virTypedParamsGetString(params, nparams, VIR_MIGRATE_PARAM_LISTEN_ADDRESS, - &listenAddress) < 0) + &listenAddress) < 0 || + virTypedParamsGetInt(params, nparams, + VIR_MIGRATE_PARAM_NBD_PORT, + &nbdPort) < 0) goto cleanup; nmigrate_disks = virTypedParamsGetStringList(params, nparams, @@ -12308,7 +12312,8 @@ qemuDomainMigratePrepare3Params(virConnectPtr dconn, cookieout, cookieoutlen, uri_in, uri_out, &def, origname, listenAddress, - nmigrate_disks, migrate_disks, flags); + nmigrate_disks, migrate_disks, + nbdPort, flags); cleanup: VIR_FREE(migrate_disks); @@ -12442,7 +12447,7 @@ qemuDomainMigratePerform3(virDomainPtr dom, } return qemuMigrationPerform(driver, dom->conn, vm, xmlin, - dconnuri, uri, NULL, NULL, 0, NULL, + dconnuri, uri, NULL, NULL, 0, NULL, 0, cookiein, cookieinlen, cookieout, cookieoutlen, flags, dname, resource, true); @@ -12469,6 +12474,7 @@ qemuDomainMigratePerform3Params(virDomainPtr dom, int nmigrate_disks; const char **migrate_disks = NULL; unsigned long long bandwidth = 0; + int nbdPort = 0; int ret = -1; virCheckFlags(QEMU_MIGRATION_FLAGS, -1); @@ -12492,7 +12498,10 @@ qemuDomainMigratePerform3Params(virDomainPtr dom, &graphicsuri) < 0 || virTypedParamsGetString(params, nparams, VIR_MIGRATE_PARAM_LISTEN_ADDRESS, - &listenAddress) < 0) + &listenAddress) < 0 || + virTypedParamsGetInt(params, nparams, + VIR_MIGRATE_PARAM_NBD_PORT, + &nbdPort) < 0) goto cleanup; nmigrate_disks = virTypedParamsGetStringList(params, nparams, @@ -12512,7 +12521,7 @@ qemuDomainMigratePerform3Params(virDomainPtr dom, ret = qemuMigrationPerform(driver, dom->conn, vm, dom_xml, dconnuri, uri, graphicsuri, listenAddress, - nmigrate_disks, migrate_disks, + nmigrate_disks, migrate_disks, nbdPort, cookiein, cookieinlen, cookieout, cookieoutlen, flags, dname, bandwidth, true); cleanup: diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c index f2c7b61..4cadf34 100644 --- a/src/qemu/qemu_migration.c +++ b/src/qemu/qemu_migration.c @@ -1709,7 +1709,8 @@ qemuMigrationStartNBDServer(virQEMUDriverPtr driver, virDomainObjPtr vm, const char *listenAddr, size_t nmigrate_disks, - const char **migrate_disks) + const char **migrate_disks, + int nbdPort) { int ret = -1; qemuDomainObjPrivatePtr priv = vm->privateData; @@ -1717,6 +1718,12 @@ qemuMigrationStartNBDServer(virQEMUDriverPtr driver, char *diskAlias = NULL; size_t i; + if (nbdPort < 0 || nbdPort > USHRT_MAX) { + virReportError(VIR_ERR_INVALID_ARG, "%s", + _("nbd port must be in range 0-65535")); + return -1; + } + for (i = 0; i < vm->def->ndisks; i++) { virDomainDiskDefPtr disk = vm->def->disks[i]; @@ -1733,10 +1740,15 @@ qemuMigrationStartNBDServer(virQEMUDriverPtr driver, QEMU_ASYNC_JOB_MIGRATION_IN) < 0) goto cleanup; - if (!port && - ((virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) || - (qemuMonitorNBDServerStart(priv->mon, listenAddr, port) < 0))) { - goto exit_monitor; + if (port == 0) { + if (nbdPort) + port = nbdPort; + else + if (virPortAllocatorAcquire(driver->migrationPorts, &port) < 0) + goto exit_monitor; + + if (qemuMonitorNBDServerStart(priv->mon, listenAddr, port) < 0) + goto exit_monitor; } if (qemuMonitorNBDServerAdd(priv->mon, diskAlias, true) < 0) @@ -1750,7 +1762,8 @@ qemuMigrationStartNBDServer(virQEMUDriverPtr driver, cleanup: VIR_FREE(diskAlias); - if (ret < 0) + // second clause means port is autoselected + if (ret < 0 && nbdPort == 0) virPortAllocatorRelease(driver->migrationPorts, port); return ret; @@ -3312,6 +3325,7 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, const char *listenAddress, size_t nmigrate_disks, const char **migrate_disks, + int nbdPort, unsigned long flags) { virDomainObjPtr vm = NULL; @@ -3513,7 +3527,8 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, flags & (VIR_MIGRATE_NON_SHARED_DISK | VIR_MIGRATE_NON_SHARED_INC) && virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_NBD_SERVER)) { if (qemuMigrationStartNBDServer(driver, vm, incoming->address, - nmigrate_disks, migrate_disks) < 0) { + nmigrate_disks, migrate_disks, + nbdPort) < 0) { goto stopjob; } cookieFlags |= QEMU_MIGRATION_COOKIE_NBD; @@ -3565,6 +3580,10 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, if (autoPort) priv->migrationPort = port; + // in this case port is autoselected and we don't need to manage it anymore + // after cookie is baked + if (nbdPort != 0) + priv->nbdPort = 0; ret = 0; cleanup: @@ -3576,7 +3595,10 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, /* priv is set right after vm is added to the list of domains * and there is no 'goto cleanup;' in the middle of those */ VIR_FREE(priv->origname); - virPortAllocatorRelease(driver->migrationPorts, priv->nbdPort); + // release if port is auto selected which is not the case if + // it is given in parameters + if (nbdPort == 0) + virPortAllocatorRelease(driver->migrationPorts, priv->nbdPort); priv->nbdPort = 0; qemuDomainRemoveInactive(driver, vm); } @@ -3633,7 +3655,7 @@ qemuMigrationPrepareTunnel(virQEMUDriverPtr driver, ret = qemuMigrationPrepareAny(driver, dconn, cookiein, cookieinlen, cookieout, cookieoutlen, def, origname, - st, NULL, 0, false, NULL, 0, NULL, flags); + st, NULL, 0, false, NULL, 0, NULL, 0, flags); return ret; } @@ -3675,6 +3697,7 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, const char *listenAddress, size_t nmigrate_disks, const char **migrate_disks, + int nbdPort, unsigned long flags) { unsigned short port = 0; @@ -3688,11 +3711,11 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, VIR_DEBUG("driver=%p, dconn=%p, cookiein=%s, cookieinlen=%d, " "cookieout=%p, cookieoutlen=%p, uri_in=%s, uri_out=%p, " "def=%p, origname=%s, listenAddress=%s, " - "nmigrate_disks=%zu, migrate_disks=%p, flags=%lx", + "nmigrate_disks=%zu, migrate_disks=%p, nbdPort=%d, flags=%lx", driver, dconn, NULLSTR(cookiein), cookieinlen, cookieout, cookieoutlen, NULLSTR(uri_in), uri_out, *def, origname, NULLSTR(listenAddress), - nmigrate_disks, migrate_disks, flags); + nmigrate_disks, migrate_disks, nbdPort, flags); *uri_out = NULL; @@ -3797,7 +3820,7 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, cookieout, cookieoutlen, def, origname, NULL, uri ? uri->scheme : "tcp", port, autoPort, listenAddress, - nmigrate_disks, migrate_disks, flags); + nmigrate_disks, migrate_disks, nbdPort, flags); cleanup: virURIFree(uri); VIR_FREE(hostname); @@ -4867,6 +4890,7 @@ doPeer2PeerMigrate3(virQEMUDriverPtr driver, const char *listenAddress, size_t nmigrate_disks, const char **migrate_disks, + int nbdPort, unsigned long long bandwidth, bool useParams, unsigned long flags) @@ -4890,11 +4914,11 @@ doPeer2PeerMigrate3(virQEMUDriverPtr driver, VIR_DEBUG("driver=%p, sconn=%p, dconn=%p, dconnuri=%s, vm=%p, xmlin=%s, " "dname=%s, uri=%s, graphicsuri=%s, listenAddress=%s, " - "nmigrate_disks=%zu, migrate_disks=%p, bandwidth=%llu, " - "useParams=%d, flags=%lx", + "nmigrate_disks=%zu, migrate_disks=%p, nbdPort=%d, " + "bandwidth=%llu, useParams=%d, flags=%lx", driver, sconn, dconn, NULLSTR(dconnuri), vm, NULLSTR(xmlin), NULLSTR(dname), NULLSTR(uri), NULLSTR(graphicsuri), - NULLSTR(listenAddress), nmigrate_disks, migrate_disks, + NULLSTR(listenAddress), nmigrate_disks, migrate_disks, nbdPort, bandwidth, useParams, flags); /* Unlike the virDomainMigrateVersion3 counterpart, we don't need @@ -4944,6 +4968,11 @@ doPeer2PeerMigrate3(virQEMUDriverPtr driver, VIR_MIGRATE_PARAM_MIGRATE_DISKS, migrate_disks[i]) < 0) goto cleanup; + if (nbdPort && + virTypedParamsAddInt(&params, &nparams, &maxparams, + VIR_MIGRATE_PARAM_NBD_PORT, + nbdPort) < 0) + goto cleanup; } if (virDomainObjGetState(vm, NULL) == VIR_DOMAIN_PAUSED) @@ -5210,6 +5239,7 @@ static int doPeer2PeerMigrate(virQEMUDriverPtr driver, const char *listenAddress, size_t nmigrate_disks, const char **migrate_disks, + int nbdPort, unsigned long flags, const char *dname, unsigned long resource, @@ -5225,10 +5255,12 @@ static int doPeer2PeerMigrate(virQEMUDriverPtr driver, VIR_DEBUG("driver=%p, sconn=%p, vm=%p, xmlin=%s, dconnuri=%s, uri=%s, " "graphicsuri=%s, listenAddress=%s, nmigrate_disks=%zu, " - "migrate_disks=%p, flags=%lx, dname=%s, resource=%lu", + "migrate_disks=%p, nbdPort=%d, flags=%lx, dname=%s, " + "resource=%lu", driver, sconn, vm, NULLSTR(xmlin), NULLSTR(dconnuri), NULLSTR(uri), NULLSTR(graphicsuri), NULLSTR(listenAddress), - nmigrate_disks, migrate_disks, flags, NULLSTR(dname), resource); + nmigrate_disks, migrate_disks, nbdPort, flags, NULLSTR(dname), + resource); if (flags & VIR_MIGRATE_TUNNELLED && uri) { virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s", @@ -5323,8 +5355,8 @@ static int doPeer2PeerMigrate(virQEMUDriverPtr driver, if (*v3proto) { ret = doPeer2PeerMigrate3(driver, sconn, dconn, dconnuri, vm, xmlin, dname, uri, graphicsuri, listenAddress, - nmigrate_disks, migrate_disks, resource, - useParams, flags); + nmigrate_disks, migrate_disks, nbdPort, + resource, useParams, flags); } else { ret = doPeer2PeerMigrate2(driver, sconn, dconn, vm, dconnuri, flags, dname, resource); @@ -5361,6 +5393,7 @@ qemuMigrationPerformJob(virQEMUDriverPtr driver, const char *listenAddress, size_t nmigrate_disks, const char **migrate_disks, + int nbdPort, const char *cookiein, int cookieinlen, char **cookieout, @@ -5396,7 +5429,7 @@ qemuMigrationPerformJob(virQEMUDriverPtr driver, if ((flags & (VIR_MIGRATE_TUNNELLED | VIR_MIGRATE_PEER2PEER))) { ret = doPeer2PeerMigrate(driver, conn, vm, xmlin, dconnuri, uri, graphicsuri, listenAddress, - nmigrate_disks, migrate_disks, + nmigrate_disks, migrate_disks, nbdPort, flags, dname, resource, &v3proto); } else { qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PERFORM2); @@ -5529,6 +5562,7 @@ qemuMigrationPerform(virQEMUDriverPtr driver, const char *listenAddress, size_t nmigrate_disks, const char **migrate_disks, + int nbdPort, const char *cookiein, int cookieinlen, char **cookieout, @@ -5540,13 +5574,14 @@ qemuMigrationPerform(virQEMUDriverPtr driver, { VIR_DEBUG("driver=%p, conn=%p, vm=%p, xmlin=%s, dconnuri=%s, " "uri=%s, graphicsuri=%s, listenAddress=%s, " - "nmigrate_disks=%zu, migrate_disks=%p, " + "nmigrate_disks=%zu, migrate_disks=%p, nbdPort=%d, " "cookiein=%s, cookieinlen=%d, cookieout=%p, cookieoutlen=%p, " "flags=%lx, dname=%s, resource=%lu, v3proto=%d", driver, conn, vm, NULLSTR(xmlin), NULLSTR(dconnuri), NULLSTR(uri), NULLSTR(graphicsuri), NULLSTR(listenAddress), - nmigrate_disks, migrate_disks, NULLSTR(cookiein), cookieinlen, - cookieout, cookieoutlen, flags, NULLSTR(dname), resource, v3proto); + nmigrate_disks, migrate_disks, nbdPort, + NULLSTR(cookiein), cookieinlen, cookieout, cookieoutlen, + flags, NULLSTR(dname), resource, v3proto); if ((flags & (VIR_MIGRATE_TUNNELLED | VIR_MIGRATE_PEER2PEER))) { if (cookieinlen) { @@ -5557,7 +5592,7 @@ qemuMigrationPerform(virQEMUDriverPtr driver, return qemuMigrationPerformJob(driver, conn, vm, xmlin, dconnuri, uri, graphicsuri, listenAddress, - nmigrate_disks, migrate_disks, + nmigrate_disks, migrate_disks, nbdPort, cookiein, cookieinlen, cookieout, cookieoutlen, flags, dname, resource, v3proto); @@ -5578,7 +5613,7 @@ qemuMigrationPerform(virQEMUDriverPtr driver, } else { return qemuMigrationPerformJob(driver, conn, vm, xmlin, NULL, uri, graphicsuri, listenAddress, - nmigrate_disks, migrate_disks, + nmigrate_disks, migrate_disks, nbdPort, cookiein, cookieinlen, cookieout, cookieoutlen, flags, dname, resource, v3proto); diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h index 2445e13..2796361 100644 --- a/src/qemu/qemu_migration.h +++ b/src/qemu/qemu_migration.h @@ -51,6 +51,7 @@ VIR_MIGRATE_PARAM_BANDWIDTH, VIR_TYPED_PARAM_ULLONG, \ VIR_MIGRATE_PARAM_GRAPHICS_URI, VIR_TYPED_PARAM_STRING, \ VIR_MIGRATE_PARAM_LISTEN_ADDRESS, VIR_TYPED_PARAM_STRING, \ + VIR_MIGRATE_PARAM_NBD_PORT, VIR_TYPED_PARAM_INT, \ VIR_MIGRATE_PARAM_MIGRATE_DISKS, VIR_TYPED_PARAM_STRING | \ VIR_TYPED_PARAM_MULTIPLE, \ NULL @@ -134,6 +135,7 @@ int qemuMigrationPrepareDirect(virQEMUDriverPtr driver, const char *listenAddress, size_t nmigrate_disks, const char **migrate_disks, + int nbdPort, unsigned long flags); int qemuMigrationPerform(virQEMUDriverPtr driver, @@ -146,6 +148,7 @@ int qemuMigrationPerform(virQEMUDriverPtr driver, const char *listenAddress, size_t nmigrate_disks, const char **migrate_disks, + int nbdPort, const char *cookiein, int cookieinlen, char **cookieout, diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c index bf65a60..e648a3a 100644 --- a/tools/virsh-domain.c +++ b/tools/virsh-domain.c @@ -9653,6 +9653,10 @@ static const vshCmdOptDef opts_migrate[] = { .type = VSH_OT_STRING, .help = N_("comma separated list of disks to be migrated") }, + {.name = "nbd-port", + .type = VSH_OT_INT, + .help = N_("port to use by target nbd server") + }, {.name = NULL} }; @@ -9663,6 +9667,7 @@ doMigrate(void *opaque) virDomainPtr dom = NULL; const char *desturi = NULL; const char *opt = NULL; + int optInt = 0; unsigned int flags = 0; virshCtrlData *data = opaque; vshControl *ctl = data->ctl; @@ -9705,6 +9710,13 @@ doMigrate(void *opaque) VIR_MIGRATE_PARAM_LISTEN_ADDRESS, opt) < 0) goto save_error; + if (vshCommandOptInt(ctl, cmd, "nbd-port", &optInt) < 0) + goto out; + if (optInt && + virTypedParamsAddInt(&params, &nparams, &maxparams, + VIR_MIGRATE_PARAM_NBD_PORT, optInt) < 0) + goto save_error; + if (vshCommandOptStringReq(ctl, cmd, "dname", &opt) < 0) goto out; if (opt && diff --git a/tools/virsh.pod b/tools/virsh.pod index 435c649..66418b2 100644 --- a/tools/virsh.pod +++ b/tools/virsh.pod @@ -1532,7 +1532,7 @@ to the I<uri> namespace is displayed instead of being modified. [I<--compressed>] [I<--abort-on-error>] [I<--auto-converge>] I<domain> I<desturi> [I<migrateuri>] [I<graphicsuri>] [I<listen-address>] [I<dname>] [I<--timeout> B<seconds>] [I<--xml> B<file>] -[I<--migrate-disks> B<disk-list>] +[I<--migrate-disks> B<disk-list>] [I<--nbd-port> B<port>] Migrate domain to another host. Add I<--live> for live migration; <--p2p> for peer-2-peer migration; I<--direct> for direct migration; or I<--tunnelled> @@ -1659,6 +1659,9 @@ addresses are accepted as well as hostnames (the resolving is done on destination). Some hypervisors do not support this feature and will return an error if this parameter is used. +Optional I<nbd-port> sets the port that hypervisor on destination side should +bind to for incoming nbd traffic. Currently it is supported only by qemu. + =item B<migrate-setmaxdowntime> I<domain> I<downtime> Set maximum tolerable downtime for a domain which is being live-migrated to -- 1.8.3.1