Re: [libvirt] [PATCH v3 21/24] qemu: command: Add support for sparse vcpu topologies

Add support for using the new approach to hotplug vcpus using device_add during startup of qemu to allow sparse vcpu topologies. There are a few limitations imposed by qemu on the supported configuration: - vcpu0 needs to be always present and not hotpluggable - non-hotpluggable cpus need to be ordered at the beginning - order of the vcpus needs to be unique for every single hotpluggable entity Qemu also doesn't really allow to query the information necessary to start a VM with the vcpus directly on the commandline. Fortunately they can be hotplugged during startup. The new hotplug code uses the following approach: - non-hotpluggable vcpus are counted and put to the -smp option - qemu is started - qemu is queried for the necessary information - the configuration is checked - the hotpluggable vcpus are hotplugged - vcpus are started This patch adds a lot of checking code and enables the support to specify the individual vcpu element with qemu. --- Notes: v3: - fixed qsort comparison function (added dereference of one level) v3: - added docs to the HTML stating qemu restrictions. docs/formatdomain.html.in | 5 + src/qemu/qemu_command.c | 20 ++- src/qemu/qemu_domain.c | 76 ++++++++- src/qemu/qemu_process.c | 178 +++++++++++++++++++++ .../qemuxml2argv-cpu-hotplug-startup.args | 20 +++ .../qemuxml2argv-cpu-hotplug-startup.xml | 29 ++++ tests/qemuxml2argvtest.c | 2 + 7 files changed, 325 insertions(+), 5 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-cpu-hotplug-startup.xml diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in index 062045b..8d3168a 100644 --- a/docs/formatdomain.html.in +++ b/docs/formatdomain.html.in @@ -580,6 +580,11 @@ Note that providing state for individual cpus may be necessary to enable support of addressable vCPU hotplug and this feature may not be supported by all hypervisors. + + For QEMU the following conditions are required. Vcpu 0 needs to be + enabled and non-hotpluggable. On PPC64 along with it vcpus that are in + the same core need to be enabled as well. All non hotpluggable cpus

+ present at boot need to be grouped after vcpu 0. <span class="since">Since 2.2.0 (QEMU only)</span> </dd> </dl> diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 28e5a7e..c1dc390 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -7082,17 +7082,29 @@ qemuBuildMachineCommandLine(virCommandPtr cmd, static int qemuBuildSmpCommandLine(virCommandPtr cmd, - const virDomainDef *def) + virDomainDefPtr def) { char *smp; virBuffer buf = VIR_BUFFER_INITIALIZER; + unsigned int maxvcpus = virDomainDefGetVcpusMax(def); + unsigned int nvcpus = 0; + virDomainVcpuDefPtr vcpu; + size_t i; + + /* count un-hotpluggable enabled vcpus. Hotpluggable ones will be added

+ * in a different way */ + for (i = 0; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(def, i); + if (vcpu->online && vcpu->hotpluggable == VIR_TRISTATE_BOOL_NO) + nvcpus++; + } virCommandAddArg(cmd, "-smp"); - virBufferAsprintf(&buf, "%u", virDomainDefGetVcpus(def)); + virBufferAsprintf(&buf, "%u", nvcpus); - if (virDomainDefHasVcpusOffline(def)) - virBufferAsprintf(&buf, ",maxcpus=%u", virDomainDefGetVcpusMax(def)); + if (nvcpus != maxvcpus) + virBufferAsprintf(&buf, ",maxcpus=%u", maxvcpus); /* sockets, cores, and threads are either all zero * or all non-zero, thus checking one of them is enough */ if (def->cpu && def->cpu->sockets) { diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index aa93498..970c34a 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -2253,6 +2253,76 @@ qemuDomainRecheckInternalPaths(virDomainDefPtr def, static int +qemuDomainDefVcpusPostParse(virDomainDefPtr def) +{ + unsigned int maxvcpus = virDomainDefGetVcpusMax(def); + virDomainVcpuDefPtr vcpu; + virDomainVcpuDefPtr prevvcpu; + size_t i; + bool has_order = false; + + /* vcpu 0 needs to be present, first, and non-hotpluggable */ + vcpu = virDomainDefGetVcpu(def, 0); + if (!vcpu->online) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("vcpu 0 can't be offline")); + return -1; + } + if (vcpu->hotpluggable == VIR_TRISTATE_BOOL_YES) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("vcpu0 can't be hotpluggable")); + return -1; + } + if (vcpu->order != 0 && vcpu->order != 1) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("vcpu0 must be enabled first")); + return -1; + } + + if (vcpu->order != 0) + has_order = true; + + prevvcpu = vcpu; + + /* all online vcpus or non online vcpu need to have order set */ + for (i = 1; i < maxvcpus; i++) { + vcpu = virDomainDefGetVcpu(def, i); + + if (vcpu->online && + (vcpu->order != 0) != has_order) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("all vcpus must have either set or unset order")); + return -1; + } + + /* few conditions for non-hotpluggable (thus online) vcpus */ + if (vcpu->hotpluggable == VIR_TRISTATE_BOOL_NO) { + /* they can be ordered only at the beinning */