qemu 2.3.0 added the -cpu host,aarch64=off option, which allows using qemu-system-aarch64 KVM to run armv7l VMs. Add a capabilities check for it, wire it up in qemu_command, and test the command line generation. --- src/qemu/qemu_capabilities.c | 7 +++++ src/qemu/qemu_capabilities.h | 1 + src/qemu/qemu_command.c | 13 ++++++++ .../qemuxml2argv-aarch64-kvm-32-on-64.args | 10 +++++++ .../qemuxml2argv-aarch64-kvm-32-on-64.xml | 35 ++++++++++++++++++++++ tests/qemuxml2argvtest.c | 9 ++++++ 6 files changed, 75 insertions(+) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 375df22..88c31fa 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -284,6 +284,7 @@ VIR_ENUM_IMPL(virQEMUCaps, QEMU_CAPS_LAST, "aes-key-wrap", "dea-key-wrap", "pci-serial", + "aarch64-off", ); @@ -3270,6 +3271,12 @@ virQEMUCapsInitQMPMonitor(virQEMUCapsPtr qemuCaps, if (qemuCaps->version >= 2002000) virQEMUCapsSet(qemuCaps, QEMU_CAPS_MACHINE_VMPORT_OPT); + /* -cpu ...,aarch64=off supported in v2.3.0 and onwards. But it + isn't detectable via qmp at this point */ + if (qemuCaps->arch == VIR_ARCH_AARCH64 && + qemuCaps->version >= 2003000) + virQEMUCapsSet(qemuCaps, QEMU_CAPS_CPU_AARCH64_OFF); + if (virQEMUCapsProbeQMPCommands(qemuCaps, mon) < 0) goto cleanup; if (virQEMUCapsProbeQMPEvents(qemuCaps, mon) < 0) diff --git a/src/qemu/qemu_capabilities.h b/src/qemu/qemu_capabilities.h index 9c956f3..b5a7770 100644 --- a/src/qemu/qemu_capabilities.h +++ b/src/qemu/qemu_capabilities.h @@ -228,6 +228,7 @@ typedef enum { QEMU_CAPS_AES_KEY_WRAP = 186, /* -machine aes_key_wrap */ QEMU_CAPS_DEA_KEY_WRAP = 187, /* -machine dea_key_wrap */ QEMU_CAPS_DEVICE_PCI_SERIAL = 188, /* -device pci-serial */ + QEMU_CAPS_CPU_AARCH64_OFF = 189, /* -cpu ...,aarch64=off */ QEMU_CAPS_LAST, /* this must always be the last item */ } virQEMUCapsFlags; diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 81e89fc..376769f 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -7024,6 +7024,19 @@ qemuBuildCpuModelArgStr(virQEMUDriverPtr driver, } virBufferAddLit(buf, "host"); + if (def->os.arch == VIR_ARCH_ARMV7L && + host->arch == VIR_ARCH_AARCH64) { + if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_CPU_AARCH64_OFF)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("QEMU binary does not support CPU " + "host-passthrough for armv7l on " + "aarch64 host")); + goto cleanup; + } + + virBufferAddLit(buf, ",aarch64=off"); + } + if (ARCH_IS_PPC64(def->os.arch) && cpu->mode == VIR_CPU_MODE_HOST_MODEL && def->cpu->model != NULL) { diff --git a/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args b/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args new file mode 100644 index 0000000..e60e234 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args @@ -0,0 +1,10 @@ +LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \ +/usr/bin/qemu-system-aarch64 -S -M virt -cpu host,aarch64=off -m 1024 -smp 1 \ +-nographic -nodefconfig -nodefaults \ +-monitor unix:/tmp/test-monitor,server,nowait \ +-boot c -kernel /arm.kernel -initrd /arm.initrd \ +-append 'console=ttyAMA0,115200n8 rw root=/dev/vda rootwait physmap.enabled=0' \ +-usb -drive file=/arm.raw,if=none,id=drive-virtio-disk0 \ +-device virtio-blk-device,drive=drive-virtio-disk0,id=virtio-disk0 \ +-device virtio-net-device,vlan=0,id=net0,mac=52:54:00:09:a4:37 \ +-net user,vlan=0,name=hostnet0 -serial pty diff --git a/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml b/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml new file mode 100644 index 0000000..c145b5f --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml @@ -0,0 +1,35 @@ +<domain type="kvm"> + <name>armtest</name> + <uuid>496d7ea8-9739-544b-4ebd-ef08be936e6a</uuid> + <memory>1048576</memory> + <currentMemory>1048576</currentMemory> + <vcpu>1</vcpu> + <os> + <type arch="armv7l" machine="virt">hvm</type> + <kernel>/arm.kernel</kernel> + <initrd>/arm.initrd</initrd> + <cmdline>console=ttyAMA0,115200n8 rw root=/dev/vda rootwait physmap.enabled=0</cmdline> + </os> + <features> + <acpi/> + <apic/> + <pae/> + </features> + <cpu mode='host-passthrough'/> + <clock offset="utc"/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>restart</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-aarch64</emulator> + <disk type='file' device='disk'> + <source file='/arm.raw'/> + <target dev='vda' bus='virtio'/> + </disk> + <interface type='user'> + <mac address='52:54:00:09:a4:37'/> + <model type='virtio'/> + </interface> + <console type='pty'/> + </devices> +</domain> diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 958f786..6d26222 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -1546,6 +1546,15 @@ mymain(void) DO_TEST("aarch64-gic", QEMU_CAPS_DEVICE, QEMU_CAPS_DRIVE, QEMU_CAPS_KVM); + driver.caps->host.cpu->arch = VIR_ARCH_AARCH64; + DO_TEST("aarch64-kvm-32-on-64", QEMU_CAPS_DEVICE, QEMU_CAPS_DRIVE, + QEMU_CAPS_NODEFCONFIG, QEMU_CAPS_DEVICE_VIRTIO_MMIO, + QEMU_CAPS_KVM, QEMU_CAPS_CPU_HOST, QEMU_CAPS_CPU_AARCH64_OFF); + DO_TEST_FAILURE("aarch64-kvm-32-on-64", QEMU_CAPS_DEVICE, QEMU_CAPS_DRIVE, + QEMU_CAPS_NODEFCONFIG, QEMU_CAPS_DEVICE_VIRTIO_MMIO, + QEMU_CAPS_KVM, QEMU_CAPS_CPU_HOST); + driver.caps->host.cpu->arch = cpuDefault->arch; + DO_TEST("kvm-pit-device", QEMU_CAPS_KVM_PIT_TICK_POLICY); DO_TEST("kvm-pit-delay", QEMU_CAPS_NO_KVM_PIT); DO_TEST("kvm-pit-device", QEMU_CAPS_NO_KVM_PIT, -- 2.4.1

We need to use qemu-system-aarch64 to run armv7l KVM VMs on an aarch64 host. --- src/qemu/qemu_capabilities.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 1e7bddb..7181865 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -723,19 +723,6 @@ virQEMUCapsFindBinaryForArch(virArch hostarch, return ret; } - -static bool -virQEMUCapsIsValidForKVM(virArch hostarch, - virArch guestarch) -{ - if (hostarch == guestarch) - return true; - if (hostarch == VIR_ARCH_X86_64 && - guestarch == VIR_ARCH_I686) - return true; - return false; -} - static int virQEMUCapsInitGuest(virCapsPtr caps, virQEMUCapsCachePtr cache, @@ -747,6 +734,7 @@ virQEMUCapsInitGuest(virCapsPtr caps, char *binary = NULL; virQEMUCapsPtr qemubinCaps = NULL; virQEMUCapsPtr kvmbinCaps = NULL; + bool native_kvm, x86_32on64_kvm, arm_32on64_kvm; int ret = -1; /* Check for existence of base emulator, or alternate base @@ -764,16 +752,30 @@ virQEMUCapsInitGuest(virCapsPtr caps, /* qemu-kvm/kvm binaries can only be used if * - host & guest arches match - * Or - * - hostarch is x86_64 and guest arch is i686 - * The latter simply needs "-cpu qemu32" + * - hostarch is x86_64 and guest arch is i686 (needs -cpu qemu32) + * - hostarch is aarch64 and guest arch is armv7l (needs -cpu aarch64=off) */ - if (virQEMUCapsIsValidForKVM(hostarch, guestarch)) { - const char *const kvmbins[] = { "/usr/libexec/qemu-kvm", /* RHEL */ - "qemu-kvm", /* Fedora */ - "kvm" }; /* Upstream .spec */ + native_kvm = (hostarch == guestarch); + x86_32on64_kvm = (hostarch == VIR_ARCH_X86_64 && + guestarch == VIR_ARCH_I686); + arm_32on64_kvm = (hostarch == VIR_ARCH_AARCH64 && + guestarch == VIR_ARCH_ARMV7L); + + if (native_kvm || x86_32on64_kvm || arm_32on64_kvm) { + const char *kvmbins[] = { + "/usr/libexec/qemu-kvm", /* RHEL */ + "qemu-kvm", /* Fedora */ + "kvm", /* Debian/Ubuntu */ + NULL, + }; + + if (arm_32on64_kvm) + kvmbins[3] = "qemu-system-aarch64"; for (i = 0; i < ARRAY_CARDINALITY(kvmbins); ++i) { + if (!kvmbins[i]) + continue; + kvmbin = virFindFileInPath(kvmbins[i]); if (!kvmbin) -- 2.4.1

On 05/28/2015 07:31 AM, Daniel P. Berrange wrote: We need this in the case you are running on an aarch64 host and have both qemu-system-arm and qemu-system-aarch64 installed. In this case, when you want to use KVM for arm32, you _have_ to use qemu-system-aarch64, qemu-system-arm does not work. What you suggest would mean that qemu-system-arm is grabbed from the caps cache. x86 doesn't have this problem because qemu-system-i386, qemu-system-x86_64 and by extension qemu-kvm can all be used to do 32-on-64 KVM. Thanks, Cole

On 06/08/2015 04:46 AM, Daniel P. Berrange wrote: Good point, I squashed in this and pushed: diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 50361fd..ca7a7c2 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -769,6 +769,14 @@ virQEMUCapsInitGuest(virCapsPtr caps, NULL, }; + /* x86 32-on-64 can be used with qemu-system-i386 and + * qemu-system-x86_64, so if we don't find a specific kvm binary, + * we can just fall back to the host arch native binary and + * everything works fine. + * + * arm is different in that 32-on-64 _only_ works with + * qemu-system-aarch64. So we have to add it to the kvmbins list + */ if (arm_32on64_kvm) kvmbins[3] = "qemu-system-aarch64"; Thanks, Cole