9:53 a.m.
From: "Michael R. Hines" <mrhines(a)us.ibm.com>
RDMA migration uses the 'setup' state in QEMU to optionally lock
all memory before the migration starts. The total time spent in
this state is exposed as VIR_DOMAIN_JOB_SETUP_TIME.
Additionally, QEMU also exports migration throughput (mbps) for both
memory and disk, so let's add them too: VIR_DOMAIN_JOB_MEMORY_BPS,
VIR_DOMAIN_JOB_DISK_BPS.
Signed-off-by: Michael R. Hines <mrhines(a)us.ibm.com>
Signed-off-by: Jiri Denemark <jdenemar(a)redhat.com>
---
Notes:
Version 3:
- changed MBPS to BPS
- added support for both memory and disk BPS
- changed BPS to ULLONG
- added code to transfer the new statistics to the destination
daemon when migration completes
- added the new parameters to virsh
include/libvirt/libvirt.h.in | 25 +++++++++++++++++++++++++
src/qemu/qemu_domain.c | 18 ++++++++++++++++++
src/qemu/qemu_migration.c | 17 +++++++++++++++++
src/qemu/qemu_monitor.h | 9 +++++++++
src/qemu/qemu_monitor_json.c | 17 +++++++++++++++++
tools/virsh-domain.c | 27 +++++++++++++++++++++++++++
6 files changed, 113 insertions(+)
diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in
index c2f9d26..702f797 100644
--- a/include/libvirt/libvirt.h.in
+++ b/include/libvirt/libvirt.h.in
@@ -4388,6 +4388,15 @@ int virDomainAbortJob(virDomainPtr dom);
#define VIR_DOMAIN_JOB_DOWNTIME "downtime"
/**
+ * VIR_DOMAIN_JOB_SETUP_TIME:
+ *
+ * virDomainGetJobStats field: total time in milliseconds spent preparing
+ * the migration in the 'setup' phase before the iterations begin, as
+ * VIR_TYPED_PARAM_ULLONG.
+ */
+#define VIR_DOMAIN_JOB_SETUP_TIME "setup_time"
+
+/**
* VIR_DOMAIN_JOB_DATA_TOTAL:
*
* virDomainGetJobStats field: total number of bytes supposed to be
@@ -4485,6 +4494,14 @@ int virDomainAbortJob(virDomainPtr dom);
#define VIR_DOMAIN_JOB_MEMORY_NORMAL_BYTES "memory_normal_bytes"
/**
+ * VIR_DOMAIN_JOB_MEMORY_BPS:
+ *
+ * virDomainGetJobStats field: network throughput used while migrating
+ * memory in Bytes per second, as VIR_TYPED_PARAM_ULLONG.
+ */
+#define VIR_DOMAIN_JOB_MEMORY_BPS "memory_bps"
+
+/**
* VIR_DOMAIN_JOB_DISK_TOTAL:
*
* virDomainGetJobStats field: as VIR_DOMAIN_JOB_DATA_TOTAL but only
@@ -4515,6 +4532,14 @@ int virDomainAbortJob(virDomainPtr dom);
#define VIR_DOMAIN_JOB_DISK_REMAINING "disk_remaining"
/**
+ * VIR_DOMAIN_JOB_DISK_BPS:
+ *
+ * virDomainGetJobStats field: network throughput used while migrating
+ * disks in Bytes per second, as VIR_TYPED_PARAM_ULLONG.
+ */
+#define VIR_DOMAIN_JOB_DISK_BPS "disk_bps"
+
+/**
* VIR_DOMAIN_JOB_COMPRESSION_CACHE:
*
* virDomainGetJobStats field: size of the cache (in bytes) used for
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index 863ab09..9b3edd7 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -304,6 +304,12 @@ qemuDomainJobInfoToParams(qemuDomainJobInfoPtr jobInfo,
status->downtime) < 0)
goto error;
+ if (status->setup_time_set &&
+ virTypedParamsAddULLong(&par, &npar, &maxpar,
+ VIR_DOMAIN_JOB_SETUP_TIME,
+ status->setup_time) < 0)
+ goto error;
+
if (virTypedParamsAddULLong(&par, &npar, &maxpar,
VIR_DOMAIN_JOB_DATA_TOTAL,
status->ram_total +
@@ -329,6 +335,12 @@ qemuDomainJobInfoToParams(qemuDomainJobInfoPtr jobInfo,
status->ram_remaining) < 0)
goto error;
+ if (status->ram_bps &&
+ virTypedParamsAddULLong(&par, &npar, &maxpar,
+ VIR_DOMAIN_JOB_MEMORY_BPS,
+ status->ram_bps) < 0)
+ goto error;
+
if (status->ram_duplicate_set) {
if (virTypedParamsAddULLong(&par, &npar, &maxpar,
VIR_DOMAIN_JOB_MEMORY_CONSTANT,
@@ -353,6 +365,12 @@ qemuDomainJobInfoToParams(qemuDomainJobInfoPtr jobInfo,
status->disk_remaining) < 0)
goto error;
+ if (status->disk_bps &&
+ virTypedParamsAddULLong(&par, &npar, &maxpar,
+ VIR_DOMAIN_JOB_DISK_BPS,
+ status->disk_bps) < 0)
+ goto error;
+
if (status->xbzrle_set) {
if (virTypedParamsAddULLong(&par, &npar, &maxpar,
VIR_DOMAIN_JOB_COMPRESSION_CACHE,
diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index ce1a5cd..d738f9b 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -636,6 +636,10 @@ qemuMigrationCookieStatisticsXMLFormat(virBufferPtr buf,
virBufferAsprintf(buf, "<%1$s>%2$llu</%1$s>\n",
VIR_DOMAIN_JOB_DOWNTIME,
status->downtime);
+ if (status->setup_time_set)
+ virBufferAsprintf(buf, "<%1$s>%2$llu</%1$s>\n",
+ VIR_DOMAIN_JOB_SETUP_TIME,
+ status->setup_time);
virBufferAsprintf(buf, "<%1$s>%2$llu</%1$s>\n",
VIR_DOMAIN_JOB_MEMORY_TOTAL,
@@ -646,6 +650,9 @@ qemuMigrationCookieStatisticsXMLFormat(virBufferPtr buf,
virBufferAsprintf(buf, "<%1$s>%2$llu</%1$s>\n",
VIR_DOMAIN_JOB_MEMORY_REMAINING,
status->ram_remaining);
+ virBufferAsprintf(buf, "<%1$s>%2$llu</%1$s>\n",
+ VIR_DOMAIN_JOB_MEMORY_BPS,
+ status->ram_bps);
if (status->ram_duplicate_set) {
virBufferAsprintf(buf, "<%1$s>%2$llu</%1$s>\n",
@@ -668,6 +675,9 @@ qemuMigrationCookieStatisticsXMLFormat(virBufferPtr buf,
virBufferAsprintf(buf, "<%1$s>%2$llu</%1$s>\n",
VIR_DOMAIN_JOB_DISK_REMAINING,
status->disk_remaining);
+ virBufferAsprintf(buf, "<%1$s>%2$llu</%1$s>\n",
+ VIR_DOMAIN_JOB_DISK_BPS,
+ status->disk_bps);
if (status->xbzrle_set) {
virBufferAsprintf(buf, "<%1$s>%2$llu</%1$s>\n",
@@ -904,6 +914,9 @@ qemuMigrationCookieStatisticsXMLParse(xmlXPathContextPtr ctxt)
if (virXPathULongLong("string(./" VIR_DOMAIN_JOB_DOWNTIME
"[1])",
ctxt, &status->downtime) == 0)
status->downtime_set = true;
+ if (virXPathULongLong("string(./" VIR_DOMAIN_JOB_SETUP_TIME
"[1])",
+ ctxt, &status->setup_time) == 0)
+ status->setup_time_set = true;
virXPathULongLong("string(./" VIR_DOMAIN_JOB_MEMORY_TOTAL
"[1])",
ctxt, &status->ram_total);
@@ -911,6 +924,8 @@ qemuMigrationCookieStatisticsXMLParse(xmlXPathContextPtr ctxt)
ctxt, &status->ram_transferred);
virXPathULongLong("string(./" VIR_DOMAIN_JOB_MEMORY_REMAINING
"[1])",
ctxt, &status->ram_remaining);
+ virXPathULongLong("string(./" VIR_DOMAIN_JOB_MEMORY_BPS "[1])",
+ ctxt, &status->ram_bps);
if (virXPathULongLong("string(./" VIR_DOMAIN_JOB_MEMORY_CONSTANT
"[1])",
ctxt, &status->ram_duplicate) == 0)
@@ -926,6 +941,8 @@ qemuMigrationCookieStatisticsXMLParse(xmlXPathContextPtr ctxt)
ctxt, &status->disk_transferred);
virXPathULongLong("string(./" VIR_DOMAIN_JOB_DISK_REMAINING
"[1])",
ctxt, &status->disk_remaining);
+ virXPathULongLong("string(./" VIR_DOMAIN_JOB_DISK_BPS "[1])",
+ ctxt, &status->disk_bps);
if (virXPathULongLong("string(./" VIR_DOMAIN_JOB_COMPRESSION_CACHE
"[1])",
ctxt, &status->xbzrle_cache_size) == 0)
diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
index 61a14d4..5fb83ff 100644
--- a/src/qemu/qemu_monitor.h
+++ b/src/qemu/qemu_monitor.h
@@ -423,10 +423,18 @@ struct _qemuMonitorMigrationStatus {
/* total or expected depending on status */
bool downtime_set;
unsigned long long downtime;
+ /*
+ * Duration of the QEMU 'setup' state.
+ * for RDMA, this may be on the order of several seconds
+ * if pinning support is requested before the migration begins.
+ */
+ bool setup_time_set;
+ unsigned long long setup_time;
unsigned long long ram_transferred;
unsigned long long ram_remaining;
unsigned long long ram_total;
+ unsigned long long ram_bps;
bool ram_duplicate_set;
unsigned long long ram_duplicate;
unsigned long long ram_normal;
@@ -435,6 +443,7 @@ struct _qemuMonitorMigrationStatus {
unsigned long long disk_transferred;
unsigned long long disk_remaining;
unsigned long long disk_total;
+ unsigned long long disk_bps;
bool xbzrle_set;
unsigned long long xbzrle_cache_size;
diff --git a/src/qemu/qemu_monitor_json.c b/src/qemu/qemu_monitor_json.c
index 1e14a95..ce3b2cd 100644
--- a/src/qemu/qemu_monitor_json.c
+++ b/src/qemu/qemu_monitor_json.c
@@ -2469,6 +2469,7 @@ qemuMonitorJSONGetMigrationStatusReply(virJSONValuePtr reply,
virJSONValuePtr ret;
const char *statusstr;
int rc;
+ double mbps;
if (!(ret = virJSONValueObjectGet(reply, "return"))) {
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
@@ -2501,6 +2502,10 @@ qemuMonitorJSONGetMigrationStatusReply(virJSONValuePtr reply,
if (rc == 0)
status->downtime_set = true;
+ if (virJSONValueObjectGetNumberUlong(ret, "setup-time",
+ &status->setup_time) == 0)
+ status->setup_time_set = true;
+
if (status->status == QEMU_MONITOR_MIGRATION_STATUS_ACTIVE ||
status->status == QEMU_MONITOR_MIGRATION_STATUS_COMPLETED) {
virJSONValuePtr ram = virJSONValueObjectGet(ret, "ram");
@@ -2532,6 +2537,12 @@ qemuMonitorJSONGetMigrationStatusReply(virJSONValuePtr reply,
return -1;
}
+ if (virJSONValueObjectGetNumberDouble(ram, "mbps", &mbps) == 0
&&
+ mbps > 0) {
+ /* mpbs from QEMU reports Mbits/s (M as in 10^6 not Mi as 2^20) */
+ status->ram_bps = mbps * (1000 * 1000 / 8);
+ }
+
if (virJSONValueObjectGetNumberUlong(ram, "duplicate",
&status->ram_duplicate) == 0)
status->ram_duplicate_set = true;
@@ -2568,6 +2579,12 @@ qemuMonitorJSONGetMigrationStatusReply(virJSONValuePtr reply,
"data was missing"));
return -1;
}
+
+ if (virJSONValueObjectGetNumberDouble(disk, "mbps", &mbps) == 0
&&
+ mbps > 0) {
+ /* mpbs from QEMU reports Mbits/s (M as in 10^6 not Mi as 2^20) */
+ status->disk_bps = mbps * (1000 * 1000 / 8);
+ }
}
virJSONValuePtr comp = virJSONValueObjectGet(ret, "xbzrle-cache");
diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c
index 435d045..105b99e 100644
--- a/tools/virsh-domain.c
+++ b/tools/virsh-domain.c
@@ -5518,6 +5518,16 @@ cmdDomjobinfo(vshControl *ctl, const vshCmd *cmd)
vshPrint(ctl, "%-17s %-.3lf %s\n", _("Memory remaining:"),
val, unit);
val = vshPrettyCapacity(info.memTotal, &unit);
vshPrint(ctl, "%-17s %-.3lf %s\n", _("Memory total:"), val,
unit);
+
+ if ((rc = virTypedParamsGetULLong(params, nparams,
+ VIR_DOMAIN_JOB_MEMORY_BPS,
+ &value)) < 0) {
+ goto save_error;
+ } else if (rc && value) {
+ val = vshPrettyCapacity(value, &unit);
+ vshPrint(ctl, "%-17s %-.3lf %s/s\n",
+ _("Memory bandwidth:"), val, unit);
+ }
}
if (info.fileTotal || info.fileRemaining || info.fileProcessed) {
@@ -5527,6 +5537,16 @@ cmdDomjobinfo(vshControl *ctl, const vshCmd *cmd)
vshPrint(ctl, "%-17s %-.3lf %s\n", _("File remaining:"), val,
unit);
val = vshPrettyCapacity(info.fileTotal, &unit);
vshPrint(ctl, "%-17s %-.3lf %s\n", _("File total:"), val,
unit);
+
+ if ((rc = virTypedParamsGetULLong(params, nparams,
+ VIR_DOMAIN_JOB_DISK_BPS,
+ &value)) < 0) {
+ goto save_error;
+ } else if (rc && value) {
+ val = vshPrettyCapacity(value, &unit);
+ vshPrint(ctl, "%-17s %-.3lf %s/s\n",
+ _("File bandwidth:"), val, unit);
+ }
}
if ((rc = virTypedParamsGetULLong(params, nparams,
@@ -5567,6 +5587,13 @@ cmdDomjobinfo(vshControl *ctl, const vshCmd *cmd)
}
if ((rc = virTypedParamsGetULLong(params, nparams,
+ VIR_DOMAIN_JOB_SETUP_TIME,
+ &value)) < 0)
+ goto save_error;
+ else if (rc)
+ vshPrint(ctl, "%-17s %-12llu ms\n", _("Setup time:"),
value);
+
+ if ((rc = virTypedParamsGetULLong(params, nparams,
VIR_DOMAIN_JOB_COMPRESSION_CACHE,
&value)) < 0) {
goto save_error;
--
2.1.0