4:09 a.m.
Extended interrupt mode allows >255 vCPUs with q35-based machine types.
https://bugzilla.redhat.com/show_bug.cgi?id=1451282
Ján Tomko (2):
conf: add eim attribute to <iommu><driver>
qemu: format eim on intel-iommu command line
docs/formatdomain.html.in | 10 +++++++
docs/schemas/domaincommon.rng | 5 ++++
src/conf/domain_conf.c | 20 ++++++++++++++
src/conf/domain_conf.h | 1 +
src/qemu/qemu_capabilities.c | 2 ++
src/qemu/qemu_capabilities.h | 1 +
src/qemu/qemu_command.c | 11 ++++++++
src/qemu/qemu_domain.c | 20 ++++++++++++++
tests/qemucapabilitiesdata/caps_2.8.0.x86_64.xml | 1 +
tests/qemucapabilitiesdata/caps_2.9.0.x86_64.xml | 1 +
.../qemuxml2argv-intel-iommu-eim.args | 19 +++++++++++++
.../qemuxml2argv-intel-iommu-eim.xml | 31 ++++++++++++++++++++++
tests/qemuxml2argvtest.c | 7 +++++
.../qemuxml2xmlout-intel-iommu-eim.xml | 1 +
tests/qemuxml2xmltest.c | 1 +
15 files changed, 131 insertions(+)
create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.args
create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.xml
create mode 120000 tests/qemuxml2xmloutdata/qemuxml2xmlout-intel-iommu-eim.xml
--
2.10.2
4:09 a.m.
New subject: [libvirt] [PATCH 1/2] conf: add eim attribute to <iommu><driver>
Add an attribute to control extended interrupt mode.
https://bugzilla.redhat.com/show_bug.cgi?id=1451282
---
docs/formatdomain.html.in | 10 +++++++
docs/schemas/domaincommon.rng | 5 ++++
src/conf/domain_conf.c | 20 ++++++++++++++
src/conf/domain_conf.h | 1 +
.../qemuxml2argv-intel-iommu-eim.xml | 31 ++++++++++++++++++++++
.../qemuxml2xmlout-intel-iommu-eim.xml | 1 +
tests/qemuxml2xmltest.c | 1 +
7 files changed, 69 insertions(+)
create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.xml
create mode 120000 tests/qemuxml2xmloutdata/qemuxml2xmlout-intel-iommu-eim.xml
diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in
index 3135db4..b5026b1 100644
--- a/docs/formatdomain.html.in
+++ b/docs/formatdomain.html.in
@@ -7436,6 +7436,16 @@ qemu-kvm -net nic,model=? /dev/null
<span class="since">Since 3.4.0</span> (QEMU/KVM
only)
</p>
</dd>
+ <dt><code>eim</code></dt>
+ <dd>
+ <p>
+ The <code>eim</code> attribute with possible values
+ <code>on</code> and <code>off</code> can be used
to
+ turn on extended interrupt mode. In combination with intremap
+ and split I/O APIC, this allows for more vCPUs to be used.
+ <span class="since">Since 3.4.0</span> (QEMU/KVM
only)
+ </p>
+ </dd>
</dl>
</dd>
</dl>
diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
index f88e84a..144c281 100644
--- a/docs/schemas/domaincommon.rng
+++ b/docs/schemas/domaincommon.rng
@@ -3959,6 +3959,11 @@
<ref name="virOnOff"/>
</attribute>
</optional>
+ <optional>
+ <attribute name="eim">
+ <ref name="virOnOff"/>
+ </attribute>
+ </optional>
</element>
</optional>
</element>
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 9eba70a..2264d96 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -14184,6 +14184,14 @@ virDomainIOMMUDefParseXML(xmlNodePtr node,
}
iommu->caching_mode = val;
}
+ VIR_FREE(tmp);
+ if ((tmp = virXPathString("string(./driver/@eim)", ctxt))) {
+ if ((val = virTristateSwitchTypeFromString(tmp)) < 0) {
+ virReportError(VIR_ERR_XML_ERROR, _("unknown eim value: %s"),
tmp);
+ goto cleanup;
+ }
+ iommu->eim = val;
+ }
ret = iommu;
iommu = NULL;
@@ -19856,6 +19864,14 @@ virDomainIOMMUDefCheckABIStability(virDomainIOMMUDefPtr src,
virTristateSwitchTypeToString(src->caching_mode));
return false;
}
+ if (src->eim != dst->eim) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("Target domain IOMMU device eim value '%s' "
+ "does not match source '%s'"),
+ virTristateSwitchTypeToString(dst->eim),
+ virTristateSwitchTypeToString(src->eim));
+ return false;
+ }
return true;
}
@@ -24199,6 +24215,10 @@ virDomainIOMMUDefFormat(virBufferPtr buf,
virBufferAsprintf(&childBuf, " caching_mode='%s'",
virTristateSwitchTypeToString(iommu->caching_mode));
}
+ if (iommu->eim != VIR_TRISTATE_SWITCH_ABSENT) {
+ virBufferAsprintf(&childBuf, " eim='%s'",
+ virTristateSwitchTypeToString(iommu->eim));
+ }
virBufferAddLit(&childBuf, "/>\n");
}
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 34a3596..83e0672 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -2211,6 +2211,7 @@ struct _virDomainIOMMUDef {
virDomainIOMMUModel model;
virTristateSwitch intremap;
virTristateSwitch caching_mode;
+ virTristateSwitch eim;
};
/*
* Guest VM main configuration
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.xml
b/tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.xml
new file mode 100644
index 0000000..8642ed3
--- /dev/null
+++ b/tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.xml
@@ -0,0 +1,31 @@
+<domain type='kvm'>
+ <name>QEMUGuest1</name>
+ <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+ <memory unit='KiB'>219100</memory>
+ <currentMemory unit='KiB'>219100</currentMemory>
+ <vcpu placement='static'>288</vcpu>
+ <os>
+ <type arch='x86_64' machine='q35'>hvm</type>
+ <boot dev='hd'/>
+ </os>
+ <features>
+ <ioapic driver='qemu'/>
+ </features>
+ <clock offset='utc'/>
+ <on_poweroff>destroy</on_poweroff>
+ <on_reboot>restart</on_reboot>
+ <on_crash>destroy</on_crash>
+ <devices>
+ <emulator>/usr/bin/qemu-system-x86_64</emulator>
+ <controller type='pci' index='0' model='pcie-root'/>
+ <controller type='sata' index='0'>
+ <address type='pci' domain='0x0000' bus='0x00'
slot='0x1f' function='0x2'/>
+ </controller>
+ <input type='mouse' bus='ps2'/>
+ <input type='keyboard' bus='ps2'/>
+ <memballoon model='none'/>
+ <iommu model='intel'>
+ <driver intremap='on' eim='on'/>
+ </iommu>
+ </devices>
+</domain>
diff --git a/tests/qemuxml2xmloutdata/qemuxml2xmlout-intel-iommu-eim.xml
b/tests/qemuxml2xmloutdata/qemuxml2xmlout-intel-iommu-eim.xml
new file mode 120000
index 0000000..9fbec36
--- /dev/null
+++ b/tests/qemuxml2xmloutdata/qemuxml2xmlout-intel-iommu-eim.xml
@@ -0,0 +1 @@
+../qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.xml
\ No newline at end of file
diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c
index fed74d0..fff13e2 100644
--- a/tests/qemuxml2xmltest.c
+++ b/tests/qemuxml2xmltest.c
@@ -1127,6 +1127,7 @@ mymain(void)
QEMU_CAPS_MACHINE_IOMMU);
DO_TEST("intel-iommu-ioapic", NONE);
DO_TEST("intel-iommu-caching-mode", NONE);
+ DO_TEST("intel-iommu-eim", NONE);
DO_TEST("cpu-check-none", NONE);
DO_TEST("cpu-check-partial", NONE);
--
2.10.2
9:25 a.m.
New subject: [libvirt] [PATCH 1/2] conf: add eim attribute to <iommu><driver>
On Thu, 2017-05-18 at 11:09 +0200, Ján Tomko wrote:
Add an attribute to control extended interrupt mode.
https://bugzilla.redhat.com/show_bug.cgi?id=1451282
[...]
@@ -14184,6 +14184,14 @@ virDomainIOMMUDefParseXML(xmlNodePtr node,
}
iommu->caching_mode = val;
}
Add an empty line here, please.
+ VIR_FREE(tmp);
+ if ((tmp = virXPathString("string(./driver/@eim)", ctxt))) {
+ if ((val = virTristateSwitchTypeFromString(tmp)) < 0) {
+ virReportError(VIR_ERR_XML_ERROR, _("unknown eim value: %s"),
tmp);
+ goto cleanup;
+ }
+ iommu->eim = val;
+ }
Reviewed-by: Andrea Bolognani <abologna(a)redhat.com>
--
Andrea Bolognani / Red Hat / Virtualization
9:46 a.m.
New subject: [libvirt] [PATCH 1/2] conf: add eim attribute to <iommu><driver>
On Thu, 2017-05-18 at 11:09 +0200, Ján Tomko wrote:
Add an attribute to control extended interrupt mode.
https://bugzilla.redhat.com/show_bug.cgi?id=1451282
[...]
@@ -7436,6 +7436,16 @@ qemu-kvm -net nic,model=? /dev/null
<span class="since">Since 3.4.0</span> (QEMU/KVM
only)
</p>
</dd>
+ <dt><code>eim</code></dt>
+ <dd>
+ <p>
+ The <code>eim</code> attribute with possible values
+ <code>on</code> and <code>off</code> can be used
to
+ turn on extended interrupt mode. In combination with intremap
+ and split I/O APIC, this allows for more vCPUs to be used.
+ <span class="since">Since 3.4.0</span> (QEMU/KVM
only)
+ </p>
+ </dd>
</dl>
</dd>
</dl>
Actually, the documentation could use some work too.
How about:
The <code>eim</code> attribute (with possible values
<code>on</code> and <code>off</code>) can be used to
configure Extended Interrupt Mode. A q35 domain with
split I/O APIC (as described in
<a href="#elementsFeatures">hypervisor features</a>),
and both interrupt remapping and EIM turned on for
the IOMMU, will be able to use more than 255 vCPUs.
<span class="since">Since 3.4.0</span> (QEMU/KVM only)
--
Andrea Bolognani / Red Hat / Virtualization
4:09 a.m.
New subject: [libvirt] [PATCH 2/2] qemu: format eim on intel-iommu command line
This option turns on extended interrupt mode,
which allows more than 255 vCPUs.
https://bugzilla.redhat.com/show_bug.cgi?id=1451282
---
src/qemu/qemu_capabilities.c | 2 ++
src/qemu/qemu_capabilities.h | 1 +
src/qemu/qemu_command.c | 11 +++++++++++
src/qemu/qemu_domain.c | 20 ++++++++++++++++++++
tests/qemucapabilitiesdata/caps_2.8.0.x86_64.xml | 1 +
tests/qemucapabilitiesdata/caps_2.9.0.x86_64.xml | 1 +
.../qemuxml2argv-intel-iommu-eim.args | 19 +++++++++++++++++++
tests/qemuxml2argvtest.c | 7 +++++++
8 files changed, 62 insertions(+)
create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.args
diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c
index 546dfd7..7ea8505 100644
--- a/src/qemu/qemu_capabilities.c
+++ b/src/qemu/qemu_capabilities.c
@@ -371,6 +371,7 @@ VIR_ENUM_IMPL(virQEMUCaps, QEMU_CAPS_LAST,
"kernel-irqchip.split",
"intel-iommu.intremap",
"intel-iommu.caching-mode",
+ "intel-iommu.eim",
);
@@ -1728,6 +1729,7 @@ static struct virQEMUCapsStringFlags
virQEMUCapsObjectPropsUSBNECXHCI[] = {
static struct virQEMUCapsStringFlags virQEMUCapsObjectPropsIntelIOMMU[] = {
{ "intremap", QEMU_CAPS_INTEL_IOMMU_INTREMAP },
{ "caching-mode", QEMU_CAPS_INTEL_IOMMU_CACHING_MODE },
+ { "eim", QEMU_CAPS_INTEL_IOMMU_EIM },
};
/* see documentation for virQEMUCapsQMPSchemaGetByPath for the query format */
diff --git a/src/qemu/qemu_capabilities.h b/src/qemu/qemu_capabilities.h
index aa99fda..eba9814 100644
--- a/src/qemu/qemu_capabilities.h
+++ b/src/qemu/qemu_capabilities.h
@@ -409,6 +409,7 @@ typedef enum {
QEMU_CAPS_MACHINE_KERNEL_IRQCHIP_SPLIT, /* -machine kernel_irqchip=split */
QEMU_CAPS_INTEL_IOMMU_INTREMAP, /* intel-iommu.intremap */
QEMU_CAPS_INTEL_IOMMU_CACHING_MODE, /* intel-iommu.caching-mode */
+ QEMU_CAPS_INTEL_IOMMU_EIM, /* intel-iommu.eim */
QEMU_CAPS_LAST /* this must always be the last item */
} virQEMUCapsFlags;
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 4c1a266..f190d51 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -6701,6 +6701,13 @@ qemuBuildIOMMUCommandLine(virCommandPtr cmd,
"with this QEMU binary"));
return -1;
}
+ if (iommu->eim != VIR_TRISTATE_SWITCH_ABSENT &&
+ !virQEMUCapsGet(qemuCaps, QEMU_CAPS_INTEL_IOMMU_EIM)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+ _("iommu: eim is not supported "
+ "with this QEMU binary"));
+ return -1;
+ }
break;
case VIR_DOMAIN_IOMMU_MODEL_LAST:
break;
@@ -6734,6 +6741,10 @@ qemuBuildIOMMUCommandLine(virCommandPtr cmd,
virBufferAsprintf(&opts, ",caching-mode=%s",
virTristateSwitchTypeToString(iommu->caching_mode));
}
+ if (iommu->eim != VIR_TRISTATE_SWITCH_ABSENT) {
+ virBufferAsprintf(&opts, ",eim=%s",
+ virTristateSwitchTypeToString(iommu->eim));
+ }
case VIR_DOMAIN_IOMMU_MODEL_LAST:
break;
}
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index 7c54f69..cd1825e 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -2663,6 +2663,9 @@ qemuDomainRecheckInternalPaths(virDomainDefPtr def,
}
+#define QEMU_MAX_VCPUS_WITHOUT_EIM 255
+
+
static int
qemuDomainDefVcpusPostParse(virDomainDefPtr def)
{
@@ -3071,6 +3074,23 @@ qemuDomainDefValidate(const virDomainDef *def,
}
}
+ if (ARCH_IS_X86(def->os.arch) &&
+ virDomainDefGetVcpusMax(def) > QEMU_MAX_VCPUS_WITHOUT_EIM) {
+ if (!qemuDomainIsQ35(def)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("more than %d vCPUs are only supported on "
+ "q35-based machine types"),
+ QEMU_MAX_VCPUS_WITHOUT_EIM);
+ goto cleanup;
+ }
+ if (!def->iommu || def->iommu->eim != VIR_TRISTATE_SWITCH_ON) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("more than %d vCPUs require extended interrupt "
+ "mode enabled on the iommu device"),
+ QEMU_MAX_VCPUS_WITHOUT_EIM);
+ }
+ }
+
if (qemuDomainDefValidateVideo(def) < 0)
goto cleanup;
diff --git a/tests/qemucapabilitiesdata/caps_2.8.0.x86_64.xml
b/tests/qemucapabilitiesdata/caps_2.8.0.x86_64.xml
index e515678..01edbc8 100644
--- a/tests/qemucapabilitiesdata/caps_2.8.0.x86_64.xml
+++ b/tests/qemucapabilitiesdata/caps_2.8.0.x86_64.xml
@@ -206,6 +206,7 @@
<flag name='kernel-irqchip'/>
<flag name='kernel-irqchip.split'/>
<flag name='intel-iommu.intremap'/>
+ <flag name='intel-iommu.eim'/>
<version>2008000</version>
<kvmVersion>0</kvmVersion>
<package> (v2.8.0)</package>
diff --git a/tests/qemucapabilitiesdata/caps_2.9.0.x86_64.xml
b/tests/qemucapabilitiesdata/caps_2.9.0.x86_64.xml
index 19fe4b7..95b04dd 100644
--- a/tests/qemucapabilitiesdata/caps_2.9.0.x86_64.xml
+++ b/tests/qemucapabilitiesdata/caps_2.9.0.x86_64.xml
@@ -214,6 +214,7 @@
<flag name='kernel-irqchip.split'/>
<flag name='intel-iommu.intremap'/>
<flag name='intel-iommu.caching-mode'/>
+ <flag name='intel-iommu.eim'/>
<version>2009000</version>
<kvmVersion>0</kvmVersion>
<package> (v2.9.0)</package>
diff --git a/tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.args
b/tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.args
new file mode 100644
index 0000000..ebf7c49
--- /dev/null
+++ b/tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.args
@@ -0,0 +1,19 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/home/test \
+USER=test \
+LOGNAME=test \
+QEMU_AUDIO_DRV=none \
+/usr/bin/qemu-system-x86_64 \
+-name QEMUGuest1 \
+-S \
+-machine q35,accel=kvm,kernel_irqchip=split \
+-m 214 \
+-smp 288,sockets=288,cores=1,threads=1 \
+-uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
+-nographic \
+-nodefaults \
+-monitor unix:/tmp/lib/domain--1-QEMUGuest1/monitor.sock,server,nowait \
+-no-acpi \
+-boot c \
+-device intel-iommu,intremap=on,eim=on
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
index 4269598..b360185 100644
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -2529,6 +2529,13 @@ mymain(void)
QEMU_CAPS_DEVICE_INTEL_IOMMU,
QEMU_CAPS_INTEL_IOMMU_INTREMAP,
QEMU_CAPS_INTEL_IOMMU_CACHING_MODE);
+ DO_TEST("intel-iommu-eim",
+ QEMU_CAPS_MACHINE_OPT,
+ QEMU_CAPS_MACHINE_KERNEL_IRQCHIP,
+ QEMU_CAPS_MACHINE_KERNEL_IRQCHIP_SPLIT,
+ QEMU_CAPS_INTEL_IOMMU_INTREMAP,
+ QEMU_CAPS_INTEL_IOMMU_EIM,
+ QEMU_CAPS_DEVICE_INTEL_IOMMU);
DO_TEST("cpu-hotplug-startup", QEMU_CAPS_QUERY_HOTPLUGGABLE_CPUS);
--
2.10.2
10:02 a.m.
New subject: [libvirt] [PATCH 2/2] qemu: format eim on intel-iommu command line
On Thu, 2017-05-18 at 11:09 +0200, Ján Tomko wrote:
This option turns on extended interrupt mode,
which allows more than 255 vCPUs.
https://bugzilla.redhat.com/show_bug.cgi?id=1451282
---
src/qemu/qemu_capabilities.c | 2 ++
src/qemu/qemu_capabilities.h | 1 +
src/qemu/qemu_command.c | 11 +++++++++++
src/qemu/qemu_domain.c | 20 ++++++++++++++++++++
tests/qemucapabilitiesdata/caps_2.8.0.x86_64.xml | 1 +
tests/qemucapabilitiesdata/caps_2.9.0.x86_64.xml | 1 +
.../qemuxml2argv-intel-iommu-eim.args | 19 +++++++++++++++++++
tests/qemuxml2argvtest.c | 7 +++++++
8 files changed, 62 insertions(+)
create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-intel-iommu-eim.args
[...]
@@ -2663,6 +2663,9 @@ qemuDomainRecheckInternalPaths(virDomainDefPtr
def,
}
+#define QEMU_MAX_VCPUS_WITHOUT_EIM 255
This define is very far from the code that actually uses it,
can you please move it right before qemuDomainDefValidate()?
You could even...
+
+
static int
qemuDomainDefVcpusPostParse(virDomainDefPtr def)
{
@@ -3071,6 +3074,23 @@ qemuDomainDefValidate(const virDomainDef *def,
}
}
... define it here, and...
+ if (ARCH_IS_X86(def->os.arch) &&
+ virDomainDefGetVcpusMax(def) > QEMU_MAX_VCPUS_WITHOUT_EIM) {
+ if (!qemuDomainIsQ35(def)) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("more than %d vCPUs are only supported on "
+ "q35-based machine types"),
+ QEMU_MAX_VCPUS_WITHOUT_EIM);
+ goto cleanup;
+ }
+ if (!def->iommu || def->iommu->eim != VIR_TRISTATE_SWITCH_ON) {
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+ _("more than %d vCPUs require extended interrupt "
+ "mode enabled on the iommu device"),
+ QEMU_MAX_VCPUS_WITHOUT_EIM);
+ }
+ }
.. undefine it here. Either solution is fine with me.
Reviewed-by: Andrea Bolognani <abologna(a)redhat.com>
--
Andrea Bolognani / Red Hat / Virtualization
2991
days inactive
2997
days old
5 comments
2 participants
participants (2)
-
Andrea Bolognani -
Ján Tomko