[libvirt] [PATCH 0/3] qemu: Allow arm 32-on-64 KVM
qemu 2.3.0 added support for enabling armv7l VMs to run on aarch64 hosts with KVM. First patch handles the special command line bit, last two patches are about advertising things in virsh capabilities. After these patches, 'virt-install --arch armv7l ...' on an aarch64 host with new enough qemu will automatically use KVM and generate a working config. Cole Robinson (3): qemu: command: Support arm 32-on-64 KVM with -cpu aarch64=off qemu: caps: qemu-system-aarch64 supports armv7l qemu: caps: Advertise arm 32-on-64 KVM option src/qemu/qemu_capabilities.c | 57 ++++++++++++++-------- src/qemu/qemu_capabilities.h | 1 + src/qemu/qemu_command.c | 13 +++++ .../qemuxml2argv-aarch64-kvm-32-on-64.args | 10 ++++ .../qemuxml2argv-aarch64-kvm-32-on-64.xml | 35 +++++++++++++ tests/qemuxml2argvtest.c | 9 ++++ 6 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml -- 2.4.1
qemu 2.3.0 added the -cpu host,aarch64=off option, which allows using qemu-system-aarch64 KVM to run armv7l VMs. Add a capabilities check for it, wire it up in qemu_command, and test the command line generation. --- src/qemu/qemu_capabilities.c | 7 +++++ src/qemu/qemu_capabilities.h | 1 + src/qemu/qemu_command.c | 13 ++++++++ .../qemuxml2argv-aarch64-kvm-32-on-64.args | 10 +++++++ .../qemuxml2argv-aarch64-kvm-32-on-64.xml | 35 ++++++++++++++++++++++ tests/qemuxml2argvtest.c | 9 ++++++ 6 files changed, 75 insertions(+) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 375df22..88c31fa 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -284,6 +284,7 @@ VIR_ENUM_IMPL(virQEMUCaps, QEMU_CAPS_LAST, "aes-key-wrap", "dea-key-wrap", "pci-serial", + "aarch64-off", ); @@ -3270,6 +3271,12 @@ virQEMUCapsInitQMPMonitor(virQEMUCapsPtr qemuCaps, if (qemuCaps->version >= 2002000) virQEMUCapsSet(qemuCaps, QEMU_CAPS_MACHINE_VMPORT_OPT); + /* -cpu ...,aarch64=off supported in v2.3.0 and onwards. But it + isn't detectable via qmp at this point */ + if (qemuCaps->arch == VIR_ARCH_AARCH64 && + qemuCaps->version >= 2003000) + virQEMUCapsSet(qemuCaps, QEMU_CAPS_CPU_AARCH64_OFF); + if (virQEMUCapsProbeQMPCommands(qemuCaps, mon) < 0) goto cleanup; if (virQEMUCapsProbeQMPEvents(qemuCaps, mon) < 0) diff --git a/src/qemu/qemu_capabilities.h b/src/qemu/qemu_capabilities.h index 9c956f3..b5a7770 100644 --- a/src/qemu/qemu_capabilities.h +++ b/src/qemu/qemu_capabilities.h @@ -228,6 +228,7 @@ typedef enum { QEMU_CAPS_AES_KEY_WRAP = 186, /* -machine aes_key_wrap */ QEMU_CAPS_DEA_KEY_WRAP = 187, /* -machine dea_key_wrap */ QEMU_CAPS_DEVICE_PCI_SERIAL = 188, /* -device pci-serial */ + QEMU_CAPS_CPU_AARCH64_OFF = 189, /* -cpu ...,aarch64=off */ QEMU_CAPS_LAST, /* this must always be the last item */ } virQEMUCapsFlags; diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 81e89fc..376769f 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -7024,6 +7024,19 @@ qemuBuildCpuModelArgStr(virQEMUDriverPtr driver, } virBufferAddLit(buf, "host"); + if (def->os.arch == VIR_ARCH_ARMV7L && + host->arch == VIR_ARCH_AARCH64) { + if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_CPU_AARCH64_OFF)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("QEMU binary does not support CPU " + "host-passthrough for armv7l on " + "aarch64 host")); + goto cleanup; + } + + virBufferAddLit(buf, ",aarch64=off"); + } + if (ARCH_IS_PPC64(def->os.arch) && cpu->mode == VIR_CPU_MODE_HOST_MODEL && def->cpu->model != NULL) { diff --git a/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args b/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args new file mode 100644 index 0000000..e60e234 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args @@ -0,0 +1,10 @@ +LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \ +/usr/bin/qemu-system-aarch64 -S -M virt -cpu host,aarch64=off -m 1024 -smp 1 \ +-nographic -nodefconfig -nodefaults \ +-monitor unix:/tmp/test-monitor,server,nowait \ +-boot c -kernel /arm.kernel -initrd /arm.initrd \ +-append 'console=ttyAMA0,115200n8 rw root=/dev/vda rootwait physmap.enabled=0' \ +-usb -drive file=/arm.raw,if=none,id=drive-virtio-disk0 \ +-device virtio-blk-device,drive=drive-virtio-disk0,id=virtio-disk0 \ +-device virtio-net-device,vlan=0,id=net0,mac=52:54:00:09:a4:37 \ +-net user,vlan=0,name=hostnet0 -serial pty diff --git a/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml b/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml new file mode 100644 index 0000000..c145b5f --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml @@ -0,0 +1,35 @@ +<domain type="kvm"> + <name>armtest</name> + <uuid>496d7ea8-9739-544b-4ebd-ef08be936e6a</uuid> + <memory>1048576</memory> + <currentMemory>1048576</currentMemory> + <vcpu>1</vcpu> + <os> + <type arch="armv7l" machine="virt">hvm</type> + <kernel>/arm.kernel</kernel> + <initrd>/arm.initrd</initrd> + <cmdline>console=ttyAMA0,115200n8 rw root=/dev/vda rootwait physmap.enabled=0</cmdline> + </os> + <features> + <acpi/> + <apic/> + <pae/> + </features> + <cpu mode='host-passthrough'/> + <clock offset="utc"/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>restart</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-aarch64</emulator> + <disk type='file' device='disk'> + <source file='/arm.raw'/> + <target dev='vda' bus='virtio'/> + </disk> + <interface type='user'> + <mac address='52:54:00:09:a4:37'/> + <model type='virtio'/> + </interface> + <console type='pty'/> + </devices> +</domain> diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 958f786..6d26222 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -1546,6 +1546,15 @@ mymain(void) DO_TEST("aarch64-gic", QEMU_CAPS_DEVICE, QEMU_CAPS_DRIVE, QEMU_CAPS_KVM); + driver.caps->host.cpu->arch = VIR_ARCH_AARCH64; + DO_TEST("aarch64-kvm-32-on-64", QEMU_CAPS_DEVICE, QEMU_CAPS_DRIVE, + QEMU_CAPS_NODEFCONFIG, QEMU_CAPS_DEVICE_VIRTIO_MMIO, + QEMU_CAPS_KVM, QEMU_CAPS_CPU_HOST, QEMU_CAPS_CPU_AARCH64_OFF); + DO_TEST_FAILURE("aarch64-kvm-32-on-64", QEMU_CAPS_DEVICE, QEMU_CAPS_DRIVE, + QEMU_CAPS_NODEFCONFIG, QEMU_CAPS_DEVICE_VIRTIO_MMIO, + QEMU_CAPS_KVM, QEMU_CAPS_CPU_HOST); + driver.caps->host.cpu->arch = cpuDefault->arch; + DO_TEST("kvm-pit-device", QEMU_CAPS_KVM_PIT_TICK_POLICY); DO_TEST("kvm-pit-delay", QEMU_CAPS_NO_KVM_PIT); DO_TEST("kvm-pit-device", QEMU_CAPS_NO_KVM_PIT, -- 2.4.1
And it always has, so advertise it similarly to i686 --- src/qemu/qemu_capabilities.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 88c31fa..1e7bddb 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -697,6 +697,14 @@ virQEMUCapsFindBinaryForArch(virArch hostarch, if (ret && !virFileIsExecutable(ret)) VIR_FREE(ret); + if (guestarch == VIR_ARCH_ARMV7L && + !ret && + hostarch == VIR_ARCH_AARCH64) { + ret = virFindFileInPath("qemu-system-aarch64"); + if (ret && !virFileIsExecutable(ret)) + VIR_FREE(ret); + } + if (guestarch == VIR_ARCH_I686 && !ret && hostarch == VIR_ARCH_X86_64) { -- 2.4.1
We need to use qemu-system-aarch64 to run armv7l KVM VMs on an aarch64 host. --- src/qemu/qemu_capabilities.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 1e7bddb..7181865 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -723,19 +723,6 @@ virQEMUCapsFindBinaryForArch(virArch hostarch, return ret; } - -static bool -virQEMUCapsIsValidForKVM(virArch hostarch, - virArch guestarch) -{ - if (hostarch == guestarch) - return true; - if (hostarch == VIR_ARCH_X86_64 && - guestarch == VIR_ARCH_I686) - return true; - return false; -} - static int virQEMUCapsInitGuest(virCapsPtr caps, virQEMUCapsCachePtr cache, @@ -747,6 +734,7 @@ virQEMUCapsInitGuest(virCapsPtr caps, char *binary = NULL; virQEMUCapsPtr qemubinCaps = NULL; virQEMUCapsPtr kvmbinCaps = NULL; + bool native_kvm, x86_32on64_kvm, arm_32on64_kvm; int ret = -1; /* Check for existence of base emulator, or alternate base @@ -764,16 +752,30 @@ virQEMUCapsInitGuest(virCapsPtr caps, /* qemu-kvm/kvm binaries can only be used if * - host & guest arches match - * Or - * - hostarch is x86_64 and guest arch is i686 - * The latter simply needs "-cpu qemu32" + * - hostarch is x86_64 and guest arch is i686 (needs -cpu qemu32) + * - hostarch is aarch64 and guest arch is armv7l (needs -cpu aarch64=off) */ - if (virQEMUCapsIsValidForKVM(hostarch, guestarch)) { - const char *const kvmbins[] = { "/usr/libexec/qemu-kvm", /* RHEL */ - "qemu-kvm", /* Fedora */ - "kvm" }; /* Upstream .spec */ + native_kvm = (hostarch == guestarch); + x86_32on64_kvm = (hostarch == VIR_ARCH_X86_64 && + guestarch == VIR_ARCH_I686); + arm_32on64_kvm = (hostarch == VIR_ARCH_AARCH64 && + guestarch == VIR_ARCH_ARMV7L); + + if (native_kvm || x86_32on64_kvm || arm_32on64_kvm) { + const char *kvmbins[] = { + "/usr/libexec/qemu-kvm", /* RHEL */ + "qemu-kvm", /* Fedora */ + "kvm", /* Debian/Ubuntu */ + NULL, + }; + + if (arm_32on64_kvm) + kvmbins[3] = "qemu-system-aarch64"; for (i = 0; i < ARRAY_CARDINALITY(kvmbins); ++i) { + if (!kvmbins[i]) + continue; + kvmbin = virFindFileInPath(kvmbins[i]); if (!kvmbin) -- 2.4.1
On Thu, May 21, 2015 at 07:03:28PM -0400, Cole Robinson wrote:
We need to use qemu-system-aarch64 to run armv7l KVM VMs on an aarch64 host. --- src/qemu/qemu_capabilities.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-)
diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 1e7bddb..7181865 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -723,19 +723,6 @@ virQEMUCapsFindBinaryForArch(virArch hostarch, return ret; }
- -static bool -virQEMUCapsIsValidForKVM(virArch hostarch, - virArch guestarch) -{ - if (hostarch == guestarch) - return true; - if (hostarch == VIR_ARCH_X86_64 && - guestarch == VIR_ARCH_I686) - return true; - return false; -} - static int virQEMUCapsInitGuest(virCapsPtr caps, virQEMUCapsCachePtr cache, @@ -747,6 +734,7 @@ virQEMUCapsInitGuest(virCapsPtr caps, char *binary = NULL; virQEMUCapsPtr qemubinCaps = NULL; virQEMUCapsPtr kvmbinCaps = NULL; + bool native_kvm, x86_32on64_kvm, arm_32on64_kvm; int ret = -1;
/* Check for existence of base emulator, or alternate base @@ -764,16 +752,30 @@ virQEMUCapsInitGuest(virCapsPtr caps,
/* qemu-kvm/kvm binaries can only be used if * - host & guest arches match - * Or - * - hostarch is x86_64 and guest arch is i686 - * The latter simply needs "-cpu qemu32" + * - hostarch is x86_64 and guest arch is i686 (needs -cpu qemu32) + * - hostarch is aarch64 and guest arch is armv7l (needs -cpu aarch64=off) */ - if (virQEMUCapsIsValidForKVM(hostarch, guestarch)) { - const char *const kvmbins[] = { "/usr/libexec/qemu-kvm", /* RHEL */ - "qemu-kvm", /* Fedora */ - "kvm" }; /* Upstream .spec */ + native_kvm = (hostarch == guestarch); + x86_32on64_kvm = (hostarch == VIR_ARCH_X86_64 && + guestarch == VIR_ARCH_I686); + arm_32on64_kvm = (hostarch == VIR_ARCH_AARCH64 && + guestarch == VIR_ARCH_ARMV7L); + + if (native_kvm || x86_32on64_kvm || arm_32on64_kvm) { + const char *kvmbins[] = { + "/usr/libexec/qemu-kvm", /* RHEL */ + "qemu-kvm", /* Fedora */ + "kvm", /* Debian/Ubuntu */ + NULL, + }; + + if (arm_32on64_kvm) + kvmbins[3] = "qemu-system-aarch64";
I'm unclear why you need to be adding this. We don't need it for the equivalent i686 with qemu-system-x86_64, as the earlier call to virQEMUCapsFindBinaryForArch() will already return the binary qemu-system-x86_64. IIUC, it should have returned the binary qemu-system-aarch64 too, so this just seems to duplicate the check for that binary. Regards, Daniel -- |: http://berrange.com -o- http://www.flickr.com/photos/dberrange/ :| |: http://libvirt.org -o- http://virt-manager.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: http://entangle-photo.org -o- http://live.gnome.org/gtk-vnc :|
On 05/28/2015 07:31 AM, Daniel P. Berrange wrote:
On Thu, May 21, 2015 at 07:03:28PM -0400, Cole Robinson wrote:
We need to use qemu-system-aarch64 to run armv7l KVM VMs on an aarch64 host. --- src/qemu/qemu_capabilities.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-)
diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 1e7bddb..7181865 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -723,19 +723,6 @@ virQEMUCapsFindBinaryForArch(virArch hostarch, return ret; }
- -static bool -virQEMUCapsIsValidForKVM(virArch hostarch, - virArch guestarch) -{ - if (hostarch == guestarch) - return true; - if (hostarch == VIR_ARCH_X86_64 && - guestarch == VIR_ARCH_I686) - return true; - return false; -} - static int virQEMUCapsInitGuest(virCapsPtr caps, virQEMUCapsCachePtr cache, @@ -747,6 +734,7 @@ virQEMUCapsInitGuest(virCapsPtr caps, char *binary = NULL; virQEMUCapsPtr qemubinCaps = NULL; virQEMUCapsPtr kvmbinCaps = NULL; + bool native_kvm, x86_32on64_kvm, arm_32on64_kvm; int ret = -1;
/* Check for existence of base emulator, or alternate base @@ -764,16 +752,30 @@ virQEMUCapsInitGuest(virCapsPtr caps,
/* qemu-kvm/kvm binaries can only be used if * - host & guest arches match - * Or - * - hostarch is x86_64 and guest arch is i686 - * The latter simply needs "-cpu qemu32" + * - hostarch is x86_64 and guest arch is i686 (needs -cpu qemu32) + * - hostarch is aarch64 and guest arch is armv7l (needs -cpu aarch64=off) */ - if (virQEMUCapsIsValidForKVM(hostarch, guestarch)) { - const char *const kvmbins[] = { "/usr/libexec/qemu-kvm", /* RHEL */ - "qemu-kvm", /* Fedora */ - "kvm" }; /* Upstream .spec */ + native_kvm = (hostarch == guestarch); + x86_32on64_kvm = (hostarch == VIR_ARCH_X86_64 && + guestarch == VIR_ARCH_I686); + arm_32on64_kvm = (hostarch == VIR_ARCH_AARCH64 && + guestarch == VIR_ARCH_ARMV7L); + + if (native_kvm || x86_32on64_kvm || arm_32on64_kvm) { + const char *kvmbins[] = { + "/usr/libexec/qemu-kvm", /* RHEL */ + "qemu-kvm", /* Fedora */ + "kvm", /* Debian/Ubuntu */ + NULL, + }; + + if (arm_32on64_kvm) + kvmbins[3] = "qemu-system-aarch64";
I'm unclear why you need to be adding this. We don't need it for the equivalent i686 with qemu-system-x86_64, as the earlier call to virQEMUCapsFindBinaryForArch() will already return the binary qemu-system-x86_64. IIUC, it should have returned the binary qemu-system-aarch64 too, so this just seems to duplicate the check for that binary.
We need this in the case you are running on an aarch64 host and have both qemu-system-arm and qemu-system-aarch64 installed. In this case, when you want to use KVM for arm32, you _have_ to use qemu-system-aarch64, qemu-system-arm does not work. What you suggest would mean that qemu-system-arm is grabbed from the caps cache. x86 doesn't have this problem because qemu-system-i386, qemu-system-x86_64 and by extension qemu-kvm can all be used to do 32-on-64 KVM. Thanks, Cole
On Sat, Jun 06, 2015 at 07:06:42PM -0400, Cole Robinson wrote:
On 05/28/2015 07:31 AM, Daniel P. Berrange wrote:
On Thu, May 21, 2015 at 07:03:28PM -0400, Cole Robinson wrote:
We need to use qemu-system-aarch64 to run armv7l KVM VMs on an aarch64 host. --- src/qemu/qemu_capabilities.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-)
diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 1e7bddb..7181865 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -723,19 +723,6 @@ virQEMUCapsFindBinaryForArch(virArch hostarch, return ret; }
- -static bool -virQEMUCapsIsValidForKVM(virArch hostarch, - virArch guestarch) -{ - if (hostarch == guestarch) - return true; - if (hostarch == VIR_ARCH_X86_64 && - guestarch == VIR_ARCH_I686) - return true; - return false; -} - static int virQEMUCapsInitGuest(virCapsPtr caps, virQEMUCapsCachePtr cache, @@ -747,6 +734,7 @@ virQEMUCapsInitGuest(virCapsPtr caps, char *binary = NULL; virQEMUCapsPtr qemubinCaps = NULL; virQEMUCapsPtr kvmbinCaps = NULL; + bool native_kvm, x86_32on64_kvm, arm_32on64_kvm; int ret = -1;
/* Check for existence of base emulator, or alternate base @@ -764,16 +752,30 @@ virQEMUCapsInitGuest(virCapsPtr caps,
/* qemu-kvm/kvm binaries can only be used if * - host & guest arches match - * Or - * - hostarch is x86_64 and guest arch is i686 - * The latter simply needs "-cpu qemu32" + * - hostarch is x86_64 and guest arch is i686 (needs -cpu qemu32) + * - hostarch is aarch64 and guest arch is armv7l (needs -cpu aarch64=off) */ - if (virQEMUCapsIsValidForKVM(hostarch, guestarch)) { - const char *const kvmbins[] = { "/usr/libexec/qemu-kvm", /* RHEL */ - "qemu-kvm", /* Fedora */ - "kvm" }; /* Upstream .spec */ + native_kvm = (hostarch == guestarch); + x86_32on64_kvm = (hostarch == VIR_ARCH_X86_64 && + guestarch == VIR_ARCH_I686); + arm_32on64_kvm = (hostarch == VIR_ARCH_AARCH64 && + guestarch == VIR_ARCH_ARMV7L); + + if (native_kvm || x86_32on64_kvm || arm_32on64_kvm) { + const char *kvmbins[] = { + "/usr/libexec/qemu-kvm", /* RHEL */ + "qemu-kvm", /* Fedora */ + "kvm", /* Debian/Ubuntu */ + NULL, + }; + + if (arm_32on64_kvm) + kvmbins[3] = "qemu-system-aarch64";
I'm unclear why you need to be adding this. We don't need it for the equivalent i686 with qemu-system-x86_64, as the earlier call to virQEMUCapsFindBinaryForArch() will already return the binary qemu-system-x86_64. IIUC, it should have returned the binary qemu-system-aarch64 too, so this just seems to duplicate the check for that binary.
We need this in the case you are running on an aarch64 host and have both qemu-system-arm and qemu-system-aarch64 installed. In this case, when you want to use KVM for arm32, you _have_ to use qemu-system-aarch64, qemu-system-arm does not work. What you suggest would mean that qemu-system-arm is grabbed from the caps cache.
x86 doesn't have this problem because qemu-system-i386, qemu-system-x86_64 and by extension qemu-kvm can all be used to do 32-on-64 KVM.
Ok, I think we need to have this explained in the code comments, because that is none obvious from reading the code & we don't want to be wondering why we did this when looking at the code again in a year :-) Regards, Daniel -- |: http://berrange.com -o- http://www.flickr.com/photos/dberrange/ :| |: http://libvirt.org -o- http://virt-manager.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: http://entangle-photo.org -o- http://live.gnome.org/gtk-vnc :|
On 06/08/2015 04:46 AM, Daniel P. Berrange wrote:
On Sat, Jun 06, 2015 at 07:06:42PM -0400, Cole Robinson wrote:
On 05/28/2015 07:31 AM, Daniel P. Berrange wrote:
On Thu, May 21, 2015 at 07:03:28PM -0400, Cole Robinson wrote:
We need to use qemu-system-aarch64 to run armv7l KVM VMs on an aarch64 host. --- src/qemu/qemu_capabilities.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-)
diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 1e7bddb..7181865 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -723,19 +723,6 @@ virQEMUCapsFindBinaryForArch(virArch hostarch, return ret; }
- -static bool -virQEMUCapsIsValidForKVM(virArch hostarch, - virArch guestarch) -{ - if (hostarch == guestarch) - return true; - if (hostarch == VIR_ARCH_X86_64 && - guestarch == VIR_ARCH_I686) - return true; - return false; -} - static int virQEMUCapsInitGuest(virCapsPtr caps, virQEMUCapsCachePtr cache, @@ -747,6 +734,7 @@ virQEMUCapsInitGuest(virCapsPtr caps, char *binary = NULL; virQEMUCapsPtr qemubinCaps = NULL; virQEMUCapsPtr kvmbinCaps = NULL; + bool native_kvm, x86_32on64_kvm, arm_32on64_kvm; int ret = -1;
/* Check for existence of base emulator, or alternate base @@ -764,16 +752,30 @@ virQEMUCapsInitGuest(virCapsPtr caps,
/* qemu-kvm/kvm binaries can only be used if * - host & guest arches match - * Or - * - hostarch is x86_64 and guest arch is i686 - * The latter simply needs "-cpu qemu32" + * - hostarch is x86_64 and guest arch is i686 (needs -cpu qemu32) + * - hostarch is aarch64 and guest arch is armv7l (needs -cpu aarch64=off) */ - if (virQEMUCapsIsValidForKVM(hostarch, guestarch)) { - const char *const kvmbins[] = { "/usr/libexec/qemu-kvm", /* RHEL */ - "qemu-kvm", /* Fedora */ - "kvm" }; /* Upstream .spec */ + native_kvm = (hostarch == guestarch); + x86_32on64_kvm = (hostarch == VIR_ARCH_X86_64 && + guestarch == VIR_ARCH_I686); + arm_32on64_kvm = (hostarch == VIR_ARCH_AARCH64 && + guestarch == VIR_ARCH_ARMV7L); + + if (native_kvm || x86_32on64_kvm || arm_32on64_kvm) { + const char *kvmbins[] = { + "/usr/libexec/qemu-kvm", /* RHEL */ + "qemu-kvm", /* Fedora */ + "kvm", /* Debian/Ubuntu */ + NULL, + }; + + if (arm_32on64_kvm) + kvmbins[3] = "qemu-system-aarch64";
I'm unclear why you need to be adding this. We don't need it for the equivalent i686 with qemu-system-x86_64, as the earlier call to virQEMUCapsFindBinaryForArch() will already return the binary qemu-system-x86_64. IIUC, it should have returned the binary qemu-system-aarch64 too, so this just seems to duplicate the check for that binary.
We need this in the case you are running on an aarch64 host and have both qemu-system-arm and qemu-system-aarch64 installed. In this case, when you want to use KVM for arm32, you _have_ to use qemu-system-aarch64, qemu-system-arm does not work. What you suggest would mean that qemu-system-arm is grabbed from the caps cache.
x86 doesn't have this problem because qemu-system-i386, qemu-system-x86_64 and by extension qemu-kvm can all be used to do 32-on-64 KVM.
Ok, I think we need to have this explained in the code comments, because that is none obvious from reading the code & we don't want to be wondering why we did this when looking at the code again in a year :-)
Good point, I squashed in this and pushed: diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c index 50361fd..ca7a7c2 100644 --- a/src/qemu/qemu_capabilities.c +++ b/src/qemu/qemu_capabilities.c @@ -769,6 +769,14 @@ virQEMUCapsInitGuest(virCapsPtr caps, NULL, }; + /* x86 32-on-64 can be used with qemu-system-i386 and + * qemu-system-x86_64, so if we don't find a specific kvm binary, + * we can just fall back to the host arch native binary and + * everything works fine. + * + * arm is different in that 32-on-64 _only_ works with + * qemu-system-aarch64. So we have to add it to the kvmbins list + */ if (arm_32on64_kvm) kvmbins[3] = "qemu-system-aarch64"; Thanks, Cole
On 05/21/2015 07:03 PM, Cole Robinson wrote:
qemu 2.3.0 added support for enabling armv7l VMs to run on aarch64 hosts with KVM. First patch handles the special command line bit, last two patches are about advertising things in virsh capabilities.
After these patches, 'virt-install --arch armv7l ...' on an aarch64 host with new enough qemu will automatically use KVM and generate a working config.
Cole Robinson (3): qemu: command: Support arm 32-on-64 KVM with -cpu aarch64=off qemu: caps: qemu-system-aarch64 supports armv7l qemu: caps: Advertise arm 32-on-64 KVM option
src/qemu/qemu_capabilities.c | 57 ++++++++++++++-------- src/qemu/qemu_capabilities.h | 1 + src/qemu/qemu_command.c | 13 +++++ .../qemuxml2argv-aarch64-kvm-32-on-64.args | 10 ++++ .../qemuxml2argv-aarch64-kvm-32-on-64.xml | 35 +++++++++++++ tests/qemuxml2argvtest.c | 9 ++++ 6 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml
Perhaps lost by having the tweak dir perms handling patch sneak in... Seems reasonable to me (ACK)... Should there be some way to document how this would be added/usable in the passthrough description in formatdomain? Or somewhere else? Not sure how anyone would know the magic incantation to define/use this... Even though this was posted before freeze, not 100% clear whether it's push-able for this release cycle. John
On 05/28/2015 07:26 AM, John Ferlan wrote:
On 05/21/2015 07:03 PM, Cole Robinson wrote:
qemu 2.3.0 added support for enabling armv7l VMs to run on aarch64 hosts with KVM. First patch handles the special command line bit, last two patches are about advertising things in virsh capabilities.
After these patches, 'virt-install --arch armv7l ...' on an aarch64 host with new enough qemu will automatically use KVM and generate a working config.
Cole Robinson (3): qemu: command: Support arm 32-on-64 KVM with -cpu aarch64=off qemu: caps: qemu-system-aarch64 supports armv7l qemu: caps: Advertise arm 32-on-64 KVM option
src/qemu/qemu_capabilities.c | 57 ++++++++++++++-------- src/qemu/qemu_capabilities.h | 1 + src/qemu/qemu_command.c | 13 +++++ .../qemuxml2argv-aarch64-kvm-32-on-64.args | 10 ++++ .../qemuxml2argv-aarch64-kvm-32-on-64.xml | 35 +++++++++++++ tests/qemuxml2argvtest.c | 9 ++++ 6 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-aarch64-kvm-32-on-64.xml
Perhaps lost by having the tweak dir perms handling patch sneak in...
Seems reasonable to me (ACK)...
Thanks, I'll wait to see if Dan has any more responses before pushing. Should there be some way to document how
this would be added/usable in the passthrough description in formatdomain? Or somewhere else? Not sure how anyone would know the magic incantation to define/use this... Even though this was posted before freeze, not 100% clear whether it's push-able for this release cycle.
The way it's meant to be consumed is just parsing the capabilities output to find supported virt-type + arch + emulator + os-type tuples, which apps already have to do. So this doesn't require any specific steps. Maybe how to _use_ the capabilities XML isn't documented anywhere but if so it's outside the scope of this patch set. Thanks, Cole
participants (3)
-
Cole Robinson -
Daniel P. Berrange -
John Ferlan