3:53 a.m.
we add a migrate status for hostdev to specify the device don't
need to initialze when VM startup, after migration end, we add
the migrate status hostdev, so can support hostdev migration.
Signed-off-by: Chen Fan <chen.fan.fnst(a)cn.fujitsu.com>
---
src/conf/domain_conf.c | 3 ++
src/conf/domain_conf.h | 7 ++++
src/qemu/qemu_command.c | 3 ++
src/qemu/qemu_driver.c | 53 +--------------------------
src/qemu/qemu_hotplug.c | 8 +++--
src/qemu/qemu_migration.c | 92 ++++++++++++++++++++++++++++++++++++++++++++---
src/qemu/qemu_migration.h | 4 +++
src/util/virhostdev.c | 3 ++
8 files changed, 114 insertions(+), 59 deletions(-)
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 7d1cd3e..b56c6fa 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -3035,6 +3035,9 @@ virDomainDeviceInfoIterateInternal(virDomainDefPtr def,
device.type = VIR_DOMAIN_DEVICE_HOSTDEV;
for (i = 0; i < def->nhostdevs; i++) {
device.data.hostdev = def->hostdevs[i];
+ if (device.data.hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+ continue;
+
if (cb(def, &device, def->hostdevs[i]->info, opaque) < 0)
return -1;
}
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 723f07b..4b7b4c9 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -543,6 +543,12 @@ struct _virDomainHostdevCaps {
} u;
};
+typedef enum {
+ VIR_DOMAIN_HOSTDEV_STATE_DEFAULT,
+ VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE,
+
+ VIR_DOMAIN_HOSTDEV_STATE_LAST
+} virDomainHostdevState;
/* basic device for direct passthrough */
struct _virDomainHostdevDef {
@@ -559,6 +565,7 @@ struct _virDomainHostdevDef {
} source;
virDomainHostdevOrigStates origstates;
virDomainDeviceInfoPtr info; /* Guest address */
+ int state;
};
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index e7e0937..dc5245a 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -10365,6 +10365,9 @@ qemuBuildCommandLine(virConnectPtr conn,
virDomainHostdevDefPtr hostdev = def->hostdevs[i];
char *devstr;
+ if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+ continue;
+
if (hostdev->info->bootIndex) {
if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS ||
(hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI
&&
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 0ba9e4a..4724171 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -12353,57 +12353,6 @@ qemuDomainMigrateBegin3(virDomainPtr domain,
cookieout, cookieoutlen, flags);
}
-static int
-qemuDomainRemovePciPassThruDevices(virConnectPtr conn,
- virDomainObjPtr vm)
-{
- virQEMUDriverPtr driver = conn->privateData;
- virDomainDeviceDef dev;
- virDomainDeviceDefPtr dev_copy = NULL;
- virCapsPtr caps = NULL;
- int ret = -1;
- size_t i;
-
- if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
- goto cleanup;
-
- if (!qemuMigrationJobIsActive(vm, QEMU_ASYNC_JOB_MIGRATION_OUT))
- goto cleanup;
-
- /* unplug passthrough bond device */
- for (i = 0; i < vm->def->nhostdevs; i++) {
- virDomainHostdevDefPtr hostdev = vm->def->hostdevs[i];
-
- if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS &&
- hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI
&&
- hostdev->source.subsys.u.pci.backend ==
VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO &&
- hostdev->source.subsys.u.pci.device == VIR_DOMAIN_HOSTDEV_PCI_DEVICE_BOND)
{
-
- dev.type = VIR_DOMAIN_DEVICE_HOSTDEV;
- dev.data.hostdev = hostdev;
-
- dev_copy = virDomainDeviceDefCopy(&dev, vm->def, caps,
driver->xmlopt);
- if (!dev_copy)
- goto cleanup;
-
- if (qemuDomainDetachHostDevice(driver, vm, dev_copy) < 0) {
- virDomainDeviceDefFree(dev_copy);
- goto cleanup;
- }
-
- virDomainDeviceDefFree(dev_copy);
- if (qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0)
- goto cleanup;
- }
- }
-
- ret = 0;
-
- cleanup:
- virObjectUnref(caps);
-
- return ret;
-}
static char *
qemuDomainMigrateBegin3Params(virDomainPtr domain,
@@ -12740,7 +12689,7 @@ qemuDomainMigratePerform3Params(virDomainPtr dom,
return -1;
}
- if (qemuDomainRemovePciPassThruDevices(dom->conn, vm) < 0) {
+ if (qemuDomainMigratePciPassThruDevices(driver, vm, false) < 0) {
qemuDomObjEndAPI(&vm);
return -1;
}
diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c
index f07c54d..13a7338 100644
--- a/src/qemu/qemu_hotplug.c
+++ b/src/qemu/qemu_hotplug.c
@@ -1239,8 +1239,9 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
unsigned int flags = 0;
- if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0)
- return -1;
+ if (hostdev->state != VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+ if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0)
+ return -1;
if (!cfg->relaxedACS)
flags |= VIR_HOSTDEV_STRICT_ACS_CHECK;
@@ -1344,7 +1345,8 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
if (ret < 0)
goto error;
- vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
+ if (hostdev->state != VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+ vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
VIR_FREE(devstr);
VIR_FREE(configfd_name);
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index 9ea83df..291cb9f 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -2001,10 +2001,7 @@ qemuMigrationIsAllowed(virQEMUDriverPtr driver, virDomainObjPtr
vm,
for (i = 0; i < def->nhostdevs; i++) {
virDomainHostdevDefPtr hostdev = def->hostdevs[i];
- if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS &&
- hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI
&&
- hostdev->source.subsys.u.pci.backend ==
VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO &&
- hostdev->source.subsys.u.pci.device ==
VIR_DOMAIN_HOSTDEV_PCI_DEVICE_BOND)
+ if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
continue;
if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS ||
@@ -2629,6 +2626,80 @@ qemuMigrationCleanup(virDomainObjPtr vm,
}
+static void
+qemuMigrationSetStateForHostdev(virDomainDefPtr def,
+ int state)
+{
+ virDomainHostdevDefPtr hostdev;
+ size_t i;
+
+ if (!def)
+ return;
+
+ for (i = 0; i < def->nhostdevs; i++) {
+ hostdev = def->hostdevs[i];
+
+ if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS &&
+ hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI
&&
+ hostdev->source.subsys.u.pci.backend ==
VIR_DOMAIN_HOSTDEV_PCI_BACKEND_VFIO &&
+ hostdev->source.subsys.u.pci.device ==
VIR_DOMAIN_HOSTDEV_PCI_DEVICE_BOND)
+ hostdev->state = state;
+ }
+}
+
+
+int
+qemuDomainMigratePciPassThruDevices(virQEMUDriverPtr driver,
+ virDomainObjPtr vm,
+ bool isPlug)
+{
+ virDomainDeviceDef dev;
+ virDomainDeviceDefPtr dev_copy = NULL;
+ virDomainHostdevDefPtr hostdev;
+ virCapsPtr caps = NULL;
+ int ret = -1;
+ int i;
+
+ if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
+ goto cleanup;
+
+ /* plug/unplug passthrough bond device */
+ for (i = vm->def->nhostdevs; i >= 0; i--) {
+ hostdev = vm->def->hostdevs[i];
+
+ if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE) {
+ if (!isPlug) {
+ dev.type = VIR_DOMAIN_DEVICE_HOSTDEV;
+ dev.data.hostdev = hostdev;
+
+ dev_copy = virDomainDeviceDefCopy(&dev, vm->def, caps,
driver->xmlopt);
+ if (!dev_copy)
+ goto cleanup;
+
+ if (qemuDomainDetachHostDevice(driver, vm, dev_copy) < 0) {
+ virDomainDeviceDefFree(dev_copy);
+ goto cleanup;
+ }
+ virDomainDeviceDefFree(dev_copy);
+ } else {
+ qemuMigrationSetStateForHostdev(vm->def,
VIR_DOMAIN_HOSTDEV_STATE_DEFAULT);
+ if (qemuDomainAttachHostDevice(NULL, driver, vm, hostdev) < 0)
+ goto cleanup;
+ }
+ if (qemuDomainUpdateDeviceList(driver, vm, QEMU_ASYNC_JOB_NONE) < 0)
+ goto cleanup;
+ }
+ }
+
+ ret = 0;
+
+ cleanup:
+ virObjectUnref(caps);
+
+ return ret;
+}
+
+
/* The caller is supposed to lock the vm and start a migration job. */
static char
*qemuMigrationBeginPhase(virQEMUDriverPtr driver,
@@ -2662,6 +2733,8 @@ static char
if (priv->job.asyncJob == QEMU_ASYNC_JOB_MIGRATION_OUT)
qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_BEGIN3);
+ qemuMigrationSetStateForHostdev(vm->def,
VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE);
+
if (!qemuMigrationIsAllowed(driver, vm, NULL, true, abort_on_error))
goto cleanup;
@@ -2885,6 +2958,8 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
goto cleanup;
+ qemuMigrationSetStateForHostdev(*def, VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE);
+
if (!qemuMigrationIsAllowed(driver, NULL, *def, true, abort_on_error))
goto cleanup;
@@ -5315,6 +5390,13 @@ qemuMigrationFinish(virQEMUDriverPtr driver,
goto endjob;
}
+ /* hotplug previous mark migrate hostdev */
+ if (qemuDomainMigratePciPassThruDevices(driver, vm, true) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("passthrough for hostdev failed"));
+ goto endjob;
+ }
+
/* Guest is successfully running, so cancel previous auto destroy */
qemuProcessAutoDestroyRemove(driver, vm);
} else if (!(flags & VIR_MIGRATE_OFFLINE)) {
@@ -5331,6 +5413,8 @@ qemuMigrationFinish(virQEMUDriverPtr driver,
VIR_WARN("Unable to encode migration cookie");
endjob:
+ qemuMigrationSetStateForHostdev(vm->def, VIR_DOMAIN_HOSTDEV_STATE_DEFAULT);
+
qemuMigrationJobFinish(driver, vm);
if (!vm->persistent && !virDomainObjIsActive(vm))
qemuDomainRemoveInactive(driver, vm);
diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h
index 1726455..fa21752 100644
--- a/src/qemu/qemu_migration.h
+++ b/src/qemu/qemu_migration.h
@@ -177,4 +177,8 @@ int qemuMigrationToFile(virQEMUDriverPtr driver, virDomainObjPtr vm,
ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(5)
ATTRIBUTE_RETURN_CHECK;
+int qemuDomainMigratePciPassThruDevices(virQEMUDriverPtr driver,
+ virDomainObjPtr vm,
+ bool isPlug);
+
#endif /* __QEMU_MIGRATION_H__ */
diff --git a/src/util/virhostdev.c b/src/util/virhostdev.c
index f583e54..4b6152a 100644
--- a/src/util/virhostdev.c
+++ b/src/util/virhostdev.c
@@ -206,6 +206,9 @@ virHostdevGetPCIHostDeviceList(virDomainHostdevDefPtr *hostdevs, int
nhostdevs)
virDomainHostdevSubsysPCIPtr pcisrc = &hostdev->source.subsys.u.pci;
virPCIDevicePtr dev;
+ if (hostdev->state == VIR_DOMAIN_HOSTDEV_STATE_READY_FOR_MIGRATE)
+ continue;
+
if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
continue;
if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
--
1.9.3