9:38 a.m.
Individual vCPU hotplug requires us to track the state of any vCPU. To
allow this add the following XML:
<domain>
...
<vcpu current='1'>2</vcpu>
<vcpus>
<vcpu id='0' enabled='no' hotpluggable='yes'/>
<vcpu id='1' enabled='yes' hotpluggable='no'
order='1'/>
</vcpus>
...
The 'enabled' attribute allows to control the state of the vcpu.
'hotpluggable' controls whether given vcpu can be hotplugged and 'order'
allows to specify the order to add the vcpus.
---
Notes:
v2:
- added documentation
- added tweak of hotplug state when reconnecting to a VM
- discussed design requests in previous thread
docs/formatdomain.html.in | 40 ++++++
docs/schemas/domaincommon.rng | 25 ++++
src/conf/domain_conf.c | 152 ++++++++++++++++++++-
src/conf/domain_conf.h | 6 +
src/qemu/qemu_domain.c | 11 +-
.../generic-vcpus-individual.xml | 23 ++++
tests/genericxml2xmltest.c | 2 +
tests/testutils.c | 4 +-
8 files changed, 259 insertions(+), 4 deletions(-)
create mode 100644 tests/genericxml2xmlindata/generic-vcpus-individual.xml
diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in
index bfbb0f2..062045b 100644
--- a/docs/formatdomain.html.in
+++ b/docs/formatdomain.html.in
@@ -489,6 +489,10 @@
<domain>
...
<vcpu placement='static' cpuset="1-4,^3,6"
current="1">2</vcpu>
+ <vcpus>
+ <vcpu id='0' enabled='yes' hotpluggable='no'
order='1'/>
+ <vcpu id='1' enabled='no' hotpluggable='yes'/>
+ </vcpus>
...
</domain>
</pre>
@@ -542,6 +546,42 @@
</dd>
</dl>
</dd>
+ <dt><code>vcpus</code></dt>
+ <dd>
+ The vcpus element allows to control state of individual vcpus.
+
+ The <code>id</code> attribute specifies the vCPU id as used by
libvirt
+ in other places such as vcpu pinning, scheduler information and NUMA
+ assignment. Note that the vcpu ID as seen in the guest may differ from
+ libvirt ID in certain cases. Valid IDs are from 0 to the maximum vcpu
+ count as set by the <code>vcpu</code> element minus 1.
+
+ The <code>enabled</code> attribute allows to control the state of
the
+ vcpu. Valid values are <code>yes</code> and
<code>no</code>.
+
+ <code>hotpluggable</code> controls whether given vcpu can be
hotplugged
+ and hotunplugged in cases when the cpu is enabled at boot. Note that
+ all disabled vcpus must be hotpluggable. Valid values are
+ <code>yes</code> and <code>no</code>.
+
+ <code>order</code> allows to specify the order to add the vcpus. For
+ hypervisors/platforms that require to insert multiple vcpus at once
+ the order may be be duplicated accross all vcpus that need to be
+ enabled at once. Specifying order is not necessary, vcpus are then
+ added in an arbitrary order.
+
+ Note that hypervisors may create hotpluggable vcpus differently from
+ boot vcpus thus special initialization may be necessary.
+
+ Hypervisors may require that vcpus enabled on boot which are not
+ hotpluggable are clustered at the beginning starting with ID 0. It may
+ be also required that vcpu 0 is always present and non-hotpluggable.
+
+ Note that providing state for individual cpus may be necessary to enable
+ support of addressable vCPU hotplug and this feature may not be
+ supported by all hypervisors.
+ <span class="since">Since 2.2.0 (QEMU only)</span>
+ </dd>
</dl>
<h3><a name="elementsIOThreadsAllocation">IOThreads
Allocation</a></h3>
diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
index 052f28c..5b3d652 100644
--- a/docs/schemas/domaincommon.rng
+++ b/docs/schemas/domaincommon.rng
@@ -583,6 +583,31 @@
</optional>
<optional>
+ <element name="vcpus">
+ <zeroOrMore>
+ <element name="vcpu">
+ <attribute name="id">
+ <ref name="unsignedInt"/>
+ </attribute>
+ <attribute name="enabled">
+ <ref name="virYesNo"/>
+ </attribute>
+ <optional>
+ <attribute name="hotpluggable">
+ <ref name="virYesNo"/>
+ </attribute>
+ </optional>
+ <optional>
+ <attribute name="order">
+ <ref name="unsignedInt"/>
+ </attribute>
+ </optional>
+ </element>
+ </zeroOrMore>
+ </element>
+ </optional>
+
+ <optional>
<element name="iothreads">
<ref name="unsignedInt"/>
</element>
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 14d4f7d..df2e53f 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -4332,6 +4332,13 @@ virDomainDefPostParseCheckFeatures(virDomainDefPtr def,
}
}
+ if (UNSUPPORTED(VIR_DOMAIN_DEF_FEATURE_INDIVIDUAL_VCPUS) &&
+ def->individualvcpus) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("individual CPU state configuration is not
supported"));
+ return -1;
+ }
+
return 0;
}
@@ -4406,6 +4413,43 @@ virDomainDefPostParseDeviceIterator(virDomainDefPtr def,
static int
+virDomainVcpuDefPostParse(virDomainDefPtr def)
+{
+ virDomainVcpuDefPtr vcpu;
+ size_t maxvcpus = virDomainDefGetVcpusMax(def);
+ size_t i;
+
+ for (i = 0; i < maxvcpus; i++) {
+ vcpu = virDomainDefGetVcpu(def, i);
+
+ switch (vcpu->hotpluggable) {
+ case VIR_TRISTATE_BOOL_ABSENT:
+ if (vcpu->online)
+ vcpu->hotpluggable = VIR_TRISTATE_BOOL_NO;
+ else
+ vcpu->hotpluggable = VIR_TRISTATE_BOOL_YES;
+ break;
+
+ case VIR_TRISTATE_BOOL_NO:
+ if (!vcpu->online) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("vcpu '%zu' is both offline and not "
+ "hotpluggable"), i);
+ return -1;
+ }
+ break;
+
+ case VIR_TRISTATE_BOOL_YES:
+ case VIR_TRISTATE_BOOL_LAST:
+ break;
+ }
+ }
+
+ return 0;
+}
+
+
+static int
virDomainDefPostParseInternal(virDomainDefPtr def,
struct virDomainDefPostParseDeviceIteratorData *data)
{
@@ -4416,6 +4460,9 @@ virDomainDefPostParseInternal(virDomainDefPtr def,
return -1;
}
+ if (virDomainVcpuDefPostParse(def) < 0)
+ return -1;
+
if (virDomainDefPostParseMemory(def, data->parseFlags) < 0)
return -1;
@@ -15521,6 +15568,8 @@ virDomainVcpuParse(virDomainDefPtr def,
virDomainXMLOptionPtr xmlopt)
{
int n;
+ xmlNodePtr *nodes = NULL;
+ size_t i;
char *tmp = NULL;
unsigned int maxvcpus;
unsigned int vcpus;
@@ -15549,8 +15598,6 @@ virDomainVcpuParse(virDomainDefPtr def,
vcpus = maxvcpus;
}
- if (virDomainDefSetVcpus(def, vcpus) < 0)
- goto cleanup;
tmp = virXPathString("string(./vcpu[1]/@placement)", ctxt);
if (tmp) {
@@ -15582,9 +15629,80 @@ virDomainVcpuParse(virDomainDefPtr def,
}
}
+ if ((n = virXPathNodeSet("./vcpus/vcpu", ctxt, &nodes)) < 0)
+ goto cleanup;
+
+ if (n) {
+ /* if individual vcpu states are provided take them as master */
+ def->individualvcpus = true;
+
+ for (i = 0; i < n; i++) {
+ virDomainVcpuDefPtr vcpu;
+ int state;
+ unsigned int id;
+ unsigned int order;
+
+ if (!(tmp = virXMLPropString(nodes[i], "id")) ||
+ virStrToLong_uip(tmp, NULL, 10, &id) < 0) {
+ virReportError(VIR_ERR_XML_ERROR, "%s",
+ _("missing or invalid vcpu id"));
+ goto cleanup;
+ }
+
+ VIR_FREE(tmp);
+
+ if (id >= def->maxvcpus) {
+ virReportError(VIR_ERR_XML_ERROR,
+ _("vcpu id '%u' is out of range of maximum
"
+ "vcpu count"), id);
+ goto cleanup;
+ }
+
+ vcpu = virDomainDefGetVcpu(def, id);
+
+ if (!(tmp = virXMLPropString(nodes[i], "enabled"))) {
+ virReportError(VIR_ERR_XML_ERROR, "%s",
+ _("missing vcpu enabled state"));
+ goto cleanup;
+ }
+
+ if ((state = virTristateBoolTypeFromString(tmp)) < 0) {
+ virReportError(VIR_ERR_XML_ERROR,
+ _("invalid vcpu 'enabled' value
'%s'"), tmp);
+ goto cleanup;
+ }
+ VIR_FREE(tmp);
+
+ vcpu->online = state == VIR_TRISTATE_BOOL_YES;
+
+ if ((tmp = virXMLPropString(nodes[i], "hotpluggable"))) {
+ if ((vcpu->hotpluggable = virTristateBoolTypeFromString(tmp)) < 0)
{
+ virReportError(VIR_ERR_XML_ERROR,
+ _("invalid vcpu 'hotpluggable' value
'%s'"), tmp);
+ goto cleanup;
+ }
+ VIR_FREE(tmp);
+ }
+
+ if ((tmp = virXMLPropString(nodes[i], "order"))) {
+ if (virStrToLong_uip(tmp, NULL, 10, &order) < 0) {
+ virReportError(VIR_ERR_XML_ERROR, "%s",
+ _("invalid vcpu order"));
+ goto cleanup;
+ }
+ vcpu->order = order;
+ VIR_FREE(tmp);
+ }
+ }
+ } else {
+ if (virDomainDefSetVcpus(def, vcpus) < 0)
+ goto cleanup;
+ }
+
ret = 0;
cleanup:
+ VIR_FREE(nodes);
VIR_FREE(tmp);
return ret;
@@ -18642,6 +18760,13 @@ virDomainDefVcpuCheckAbiStability(virDomainDefPtr src,
"destination definitions"), i);
return false;
}
+
+ if (svcpu->order != dvcpu->order) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("vcpu enable order of vCPU '%zu' differs
between "
+ "source and destination definitions"), i);
+ return false;
+ }
}
return true;
@@ -22946,6 +23071,8 @@ static int
virDomainCpuDefFormat(virBufferPtr buf,
const virDomainDef *def)
{
+ virDomainVcpuDefPtr vcpu;
+ size_t i;
char *cpumask = NULL;
int ret = -1;
@@ -22962,6 +23089,27 @@ virDomainCpuDefFormat(virBufferPtr buf,
virBufferAsprintf(buf, " current='%u'",
virDomainDefGetVcpus(def));
virBufferAsprintf(buf, ">%u</vcpu>\n",
virDomainDefGetVcpusMax(def));
+ if (def->individualvcpus) {
+ virBufferAddLit(buf, "<vcpus>\n");
+ virBufferAdjustIndent(buf, 2);
+ for (i = 0; i < def->maxvcpus; i++) {
+ vcpu = def->vcpus[i];
+
+ virBufferAsprintf(buf, "<vcpu id='%zu'
enabled='%s'",
+ i, vcpu->online ? "yes" : "no");
+ if (vcpu->hotpluggable)
+ virBufferAsprintf(buf, " hotpluggable='%s'",
+ virTristateBoolTypeToString(vcpu->hotpluggable));
+
+ if (vcpu->order != 0)
+ virBufferAsprintf(buf, " order='%d'", vcpu->order);
+
+ virBufferAddLit(buf, "/>\n");
+ }
+ virBufferAdjustIndent(buf, -2);
+ virBufferAddLit(buf, "</vcpus>\n");
+ }
+
ret = 0;
cleanup:
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 8b26724..b889125 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -2046,6 +2046,9 @@ typedef virDomainVcpuDef *virDomainVcpuDefPtr;
struct _virDomainVcpuDef {
bool online;
+ virTristateBool hotpluggable;
+ unsigned int order;
+
virBitmapPtr cpumask;
virDomainThreadSchedParam sched;
@@ -2142,6 +2145,8 @@ struct _virDomainDef {
virDomainVcpuDefPtr *vcpus;
size_t maxvcpus;
+ /* set if the vcpu definition was specified individually */
+ bool individualvcpus;
int placement_mode;
virBitmapPtr cpumask;
@@ -2344,6 +2349,7 @@ typedef enum {
VIR_DOMAIN_DEF_FEATURE_MEMORY_HOTPLUG = (1 << 1),
VIR_DOMAIN_DEF_FEATURE_OFFLINE_VCPUPIN = (1 << 2),
VIR_DOMAIN_DEF_FEATURE_NAME_SLASH = (1 << 3),
+ VIR_DOMAIN_DEF_FEATURE_INDIVIDUAL_VCPUS = (1 << 4),
} virDomainDefFeatures;
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index d4f472b..5f4c642 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -5846,8 +5846,17 @@ qemuDomainRefreshVcpuInfo(virQEMUDriverPtr driver,
VIR_STEAL_PTR(vcpupriv->alias, info[i].alias);
vcpupriv->enable_id = info[i].id;
- if (hotplug && state)
+ if (hotplug && state) {
vcpu->online = !!info[i].qom_path;
+
+ /* mark cpus that don't have an alias as non-hotpluggable */
+ if (vcpu->online) {
+ if (vcpupriv->alias)
+ vcpu->hotpluggable = VIR_TRISTATE_BOOL_YES;
+ else
+ vcpu->hotpluggable = VIR_TRISTATE_BOOL_NO;
+ }
+ }
}
ret = 0;
diff --git a/tests/genericxml2xmlindata/generic-vcpus-individual.xml
b/tests/genericxml2xmlindata/generic-vcpus-individual.xml
new file mode 100644
index 0000000..cbcf8fd
--- /dev/null
+++ b/tests/genericxml2xmlindata/generic-vcpus-individual.xml
@@ -0,0 +1,23 @@
+<domain type='qemu'>
+ <name>foobar</name>
+ <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+ <memory unit='KiB'>219136</memory>
+ <currentMemory unit='KiB'>219136</currentMemory>
+ <vcpu placement='static' current='2'>4</vcpu>
+ <vcpus>
+ <vcpu id='0' enabled='no' hotpluggable='yes'
order='1'/>
+ <vcpu id='1' enabled='yes' hotpluggable='no'/>
+ <vcpu id='2' enabled='no' hotpluggable='yes'
order='2'/>
+ <vcpu id='3' enabled='yes' hotpluggable='no'/>
+ </vcpus>
+ <os>
+ <type arch='i686' machine='pc'>hvm</type>
+ <boot dev='hd'/>
+ </os>
+ <clock offset='utc'/>
+ <on_poweroff>destroy</on_poweroff>
+ <on_reboot>restart</on_reboot>
+ <on_crash>destroy</on_crash>
+ <devices>
+ </devices>
+</domain>
diff --git a/tests/genericxml2xmltest.c b/tests/genericxml2xmltest.c
index a487727..2ea2396 100644
--- a/tests/genericxml2xmltest.c
+++ b/tests/genericxml2xmltest.c
@@ -97,6 +97,8 @@ mymain(void)
DO_TEST("perf");
+ DO_TEST("vcpus-individual");
+
virObjectUnref(caps);
virObjectUnref(xmlopt);
diff --git a/tests/testutils.c b/tests/testutils.c
index 8af8707..8ea6ab8 100644
--- a/tests/testutils.c
+++ b/tests/testutils.c
@@ -1089,7 +1089,9 @@ virCapsPtr virTestGenericCapsInit(void)
return NULL;
}
-static virDomainDefParserConfig virTestGenericDomainDefParserConfig;
+static virDomainDefParserConfig virTestGenericDomainDefParserConfig = {
+ .features = VIR_DOMAIN_DEF_FEATURE_INDIVIDUAL_VCPUS,
+};
static virDomainXMLPrivateDataCallbacks virTestGenericPrivateDataCallbacks;
virDomainXMLOptionPtr virTestGenericDomainXMLConfInit(void)
--
2.8.2