Re: librbd encryption and guest XML

On 6/17/24 02:02, Daniel P. Berrangé wrote: Hi, thank you for replying! I did some more debugging over the weekend and finally found the problem was that I had needed to resize the unencrypted raw image first before cloning it to account for the LUKS header, otherwise the image content ends up truncated after formatting and the guest unbootable. This concept is actually shown in an example in the docs https://docs.ceph.com/en/reef/rbd/rbd-encryption/#examples but I still had not made the connection between the guest not booting and the image size. So the working steps I currently have are: 1. Start with an unencrypted raw image with an OS 2. Resize the unencrypted raw image larger (example: current size + 1G) 3. Make a clone of the unencrypted image (snapshot + protect + clone) 4. Resize the unencrypted raw image back down to its original size (current_size) 5. Format the clone image with encryption 6. Resize the clone image down to the original image size (current_size) 7. Create guest XML with the encrypted clone The generated QEMU command line (which I assume has nothing wrong in it) for the unbootable guest was: 2024-06-15 01:32:00.122+0000: starting up libvirt version: 9.6.0, package: 9.6.0-1ubuntu1.1 (Ubuntu), qemu version: 8.0.4Debian 1:8.0.4+dfsg-1ubuntu3.23.10.5, kernel: 6.5.0-35-generic, hostname: controller LC_ALL=C \ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin \ HOME=/var/lib/libvirt/qemu/domain-4-instance-00000005 \ XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-4-instance-00000005/.local/share \ XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-4-instance-00000005/.cache \ XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-4-instance-00000005/.config \ /usr/bin/qemu-system-x86_64 \ -name guest=instance-00000005,debug-threads=on \ -S \ -object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-4-instance-00000005/master-key.aes"}' \ -machine pc-i440fx-8.0,usb=off,dump-guest-core=off,memory-backend=pc.ram,acpi=on \ -accel tcg \ -cpu Nehalem \ -m size=524288k \ -object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":536870912}' \ -overcommit mem-lock=off \ -smp 1,sockets=1,dies=1,cores=1,threads=1 \ -uuid 3f863740-3d6e-4469-aa34-035cd7384f9e \ -smbios 'type=1,manufacturer=OpenStack Foundation,product=OpenStack Nova,version=29.1.0,serial=3f863740-3d6e-4469-aa34-035cd7384f9e,uuid=3f863740-3d6e-4469-aa34-035cd7384f9e,family=Virtual Machine' \ -no-user-config \ -nodefaults \ -chardev socket,id=charmonitor,fd=29,server=on,wait=off \ -mon chardev=charmonitor,id=monitor,mode=control \ -rtc base=utc \ -no-shutdown \ -boot strict=on \ -object '{"qom-type":"secret","id":"libvirt-1-storage-auth-secret0","data":"VaS1qYDjnD5GACQOf+PRhdeH9MPizc4/K1jPJ3LCo0c=","keyid":"masterKey0","iv":"Hv/U+wms23aTK0GLnwn9Aw==","format":"base64"}' \ -object '{"qom-type":"secret","id":"libvirt-1-format-encryption-secret0","data":"KQn7lDD7TaPBdBHBPhIR8ytmRcPzVXjjdXbxlHhurSZceM8iYUyBgZwhoLADBD9M2v3vPvBiyFHH4PFhBjjjXC6cd8Zj8m3gI9gPkoqvkmCFh/LNGuDmeh2hIw6Ts0iO","keyid":"masterKey0","iv":"0ByebFRNgtGZcN7JarPKnQ==","format":"base64"}' \ -blockdev '{"driver":"rbd","pool":"vms","image":"3f863740-3d6e-4469-aa34-035cd7384f9e_disk","server":[{"host":"127.0.0.1","port":"6789"}],"encrypt":{"format":"luks","key-secret":"libvirt-1-format-encryption-secret0"},"user":"cinder","auth-client-required":["cephx","none"],"key-secret":"libvirt-1-storage-auth-secret0","node-name":"libvirt-1-storage","cache":{"direct":false,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \ -blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":false,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \ -device '{"driver":"virtio-blk-pci","bus":"pci.0","addr":"0x4","drive":"libvirt-1-format","id":"virtio-disk0","bootindex":1,"write-cache":"on"}' \ -netdev '{"type":"tap","fd":"32","id":"hostnet0"}' \ -device '{"driver":"virtio-net-pci","host_mtu":1442,"netdev":"hostnet0","id":"net0","mac":"fa:16:3e:48:7b:21","bus":"pci.0","addr":"0x3"}' \ -add-fd set=0,fd=30,opaque=serial0-log \ -chardev pty,id=charserial0,logfile=/dev/fdset/0,logappend=on \ -device '{"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0}' \ -audiodev '{"id":"audio1","driver":"none"}' \ -vnc 0.0.0.0:0,audiodev=audio1 \ -device '{"driver":"virtio-vga","id":"video0","max_outputs":1,"bus":"pci.0","addr":"0x2"}' \ -device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.0","addr":"0x5"}' \ -object '{"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"}' \ -device '{"driver":"virtio-rng-pci","rng":"objrng0","id":"rng0","bus":"pci.0","addr":"0x6"}' \ -device '{"driver":"vmcoreinfo"}' \ -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \ -msg timestamp=on char device redirected to /dev/pts/4 (label charserial0)