Signed-off-by: Ján Tomko <jtomko(a)redhat.com&gt; --- src/qemu/qemu_command.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index be9839cc64..cbe601099f 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -2689,8 +2689,20 @@ qemuBuildFilesystemCommandLine(virCommandPtr cmd, size_t i; for (i = 0; i < def->nfss; i++) { - if (qemuBuildFSDevCommandLine(cmd, def->fss[i], def, qemuCaps) < 0) - return -1; + switch ((virDomainFSDriverType) def->fss[i]->fsdriver) { + case VIR_DOMAIN_FS_DRIVER_TYPE_DEFAULT: + case VIR_DOMAIN_FS_DRIVER_TYPE_PATH: + case VIR_DOMAIN_FS_DRIVER_TYPE_HANDLE: + /* these drivers are handled by virtio-9p-pci */ + if (qemuBuildFSDevCommandLine(cmd, def->fss[i], def, qemuCaps) < 0) + return -1; + break; + case VIR_DOMAIN_FS_DRIVER_TYPE_LOOP: + case VIR_DOMAIN_FS_DRIVER_TYPE_NBD: + case VIR_DOMAIN_FS_DRIVER_TYPE_PLOOP: + case VIR_DOMAIN_FS_DRIVER_TYPE_LAST: + break; + } } return 0; -- 2.21.0

[to be squashed in with the actual XML addition] --- .../vhost-user-fs-fd-memory.xml | 32 +++++++++ .../vhost-user-fs-hugepages.xml | 67 +++++++++++++++++++ .../vhost-user-fs-fd-memory.x86_64-latest.xml | 1 + .../vhost-user-fs-hugepages.x86_64-latest.xml | 1 + tests/qemuxml2xmltest.c | 3 + 5 files changed, 104 insertions(+) create mode 100644 tests/qemuxml2argvdata/vhost-user-fs-fd-memory.xml create mode 100644 tests/qemuxml2argvdata/vhost-user-fs-hugepages.xml create mode 120000 tests/qemuxml2xmloutdata/vhost-user-fs-fd-memory.x86_64-latest.xml create mode 120000 tests/qemuxml2xmloutdata/vhost-user-fs-hugepages.x86_64-latest.xml diff --git a/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.xml b/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.xml new file mode 100644 index 0000000000..284ee329b3 --- /dev/null +++ b/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.xml @@ -0,0 +1,32 @@ +<domain type='kvm'> + <name>guest</name> + <uuid>126f2720-6f8e-45ab-a886-ec9277079a67</uuid> + <memory unit='KiB'>14680064</memory> + <currentMemory unit='KiB'>14680064</currentMemory> + <memoryBacking> + <source type='file'/> + <access mode='shared'/> + </memoryBacking> + <vcpu placement='static'>2</vcpu> + <os> + <type arch='x86_64' machine='pc'>hvm</type> + <boot dev='hd'/> + </os> + <cpu> + <numa> + <cell id='0' cpus='0-1' memory='14680064' unit='KiB' memAccess='shared'/> + </numa> + </cpu> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-x86_64</emulator> + <controller type='usb' index='0' model='none'/> + <controller type='pci' index='0' model='pci-root'/> + <input type='mouse' bus='ps2'/> + <input type='keyboard' bus='ps2'/> + <memballoon model='none'/> + </devices> +</domain> diff --git a/tests/qemuxml2argvdata/vhost-user-fs-hugepages.xml b/tests/qemuxml2argvdata/vhost-user-fs-hugepages.xml new file mode 100644 index 0000000000..0633394028 --- /dev/null +++ b/tests/qemuxml2argvdata/vhost-user-fs-hugepages.xml @@ -0,0 +1,67 @@ +<domain type='qemu'> + <name>guest</name> + <uuid>1ccfd97d-5eb4-478a-bbe6-88d254c16db7</uuid> + <memory unit='KiB'>4194304</memory> + <currentMemory unit='KiB'>4194304</currentMemory> + <memoryBacking> + <hugepages> + <page size='2048' unit='KiB'/> + </hugepages> + <access mode='shared'/> + </memoryBacking> + <vcpu placement='static'>2</vcpu> + <os> + <type arch='x86_64' machine='q35'>hvm</type> + <boot dev='hd'/> + </os> + <features> + <acpi/> + <apic/> + </features> + <cpu> + <numa> + <cell id='0' cpus='0-1' memory='2097152' unit='KiB' memAccess='shared'/> + </numa> + </cpu> + <clock offset='utc'/> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>destroy</on_crash> + <devices> + <emulator>/usr/bin/qemu-system-x86_64</emulator> + <disk type='file' device='disk'> + <driver name='qemu' type='qcow2'/> + <source file='/var/lib/libvirt/images/guest.qcow2'/> + <target dev='vda' bus='virtio'/> + <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/> + </disk> + <controller type='usb' index='0' model='none'/> + <controller type='sata' index='0'> + <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/> + </controller> + <controller type='pci' index='0' model='pcie-root'/> + <controller type='pci' index='1' model='pcie-root-port'> + <model name='pcie-root-port'/> + <target chassis='1' port='0x8'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0' multifunction='on'/> + </controller> + <controller type='pci' index='2' model='pcie-root-port'> + <model name='pcie-root-port'/> + <target chassis='2' port='0x9'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/> + </controller> + <controller type='pci' index='3' model='pcie-root-port'> + <model name='pcie-root-port'/> + <target chassis='3' port='0xa'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/> + </controller> + <controller type='pci' index='4' model='pcie-root-port'> + <model name='pcie-root-port'/> + <target chassis='4' port='0xb'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x3'/> + </controller> + <input type='mouse' bus='ps2'/> + <input type='keyboard' bus='ps2'/> + <memballoon model='none'/> + </devices> +</domain> diff --git a/tests/qemuxml2xmloutdata/vhost-user-fs-fd-memory.x86_64-latest.xml b/tests/qemuxml2xmloutdata/vhost-user-fs-fd-memory.x86_64-latest.xml new file mode 120000 index 0000000000..fbc552ef94 --- /dev/null +++ b/tests/qemuxml2xmloutdata/vhost-user-fs-fd-memory.x86_64-latest.xml @@ -0,0 +1 @@ +../qemuxml2argvdata/vhost-user-fs-fd-memory.xml \ No newline at end of file diff --git a/tests/qemuxml2xmloutdata/vhost-user-fs-hugepages.x86_64-latest.xml b/tests/qemuxml2xmloutdata/vhost-user-fs-hugepages.x86_64-latest.xml new file mode 120000 index 0000000000..0c0f05b254 --- /dev/null +++ b/tests/qemuxml2xmloutdata/vhost-user-fs-hugepages.x86_64-latest.xml @@ -0,0 +1 @@ +../qemuxml2argvdata/vhost-user-fs-hugepages.xml \ No newline at end of file diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c index 89c9494e6c..fa999e0094 100644 --- a/tests/qemuxml2xmltest.c +++ b/tests/qemuxml2xmltest.c @@ -1315,6 +1315,9 @@ mymain(void) DO_TEST("vhost-vsock-ccw-auto", QEMU_CAPS_DEVICE_VHOST_VSOCK, QEMU_CAPS_CCW); + DO_TEST_CAPS_LATEST("vhost-user-fs-fd-memory"); + DO_TEST_CAPS_LATEST("vhost-user-fs-hugepages"); + DO_TEST("riscv64-virt", QEMU_CAPS_DEVICE_VIRTIO_MMIO); DO_TEST("riscv64-virt-pci", -- 2.21.0

Introduce a new 'virtio-fs' driver type for filesystem. <filesystem type='mount' accessmode='passthrough'> <driver type='virtio-fs'/> <source dir='/path'/> <target dir='/path'/> <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/> </filesystem> Signed-off-by: Ján Tomko <jtomko(a)redhat.com&gt; --- docs/schemas/domaincommon.rng | 1 + src/conf/domain_conf.c | 1 + src/conf/domain_conf.h | 1 + src/qemu/qemu_command.c | 3 +++ src/qemu/qemu_domain.c | 4 ++++ src/qemu/qemu_domain_address.c | 3 +++ tests/qemuxml2argvdata/vhost-user-fs-fd-memory.xml | 6 ++++++ 7 files changed, 19 insertions(+) diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index e06f892da3..f6479c95a7 100644 --- a/docs/schemas/domaincommon.rng +++ b/docs/schemas/domaincommon.rng @@ -2567,6 +2567,7 @@ <value>loop</value> <value>nbd</value> <value>ploop</value> + <value>virtio-fs</value> </choice> </attribute> </optional> diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index d5aba7336f..2694e4bb68 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -476,6 +476,7 @@ VIR_ENUM_IMPL(virDomainFSDriver, "loop", "nbd", "ploop", + "virtio-fs", ); VIR_ENUM_IMPL(virDomainFSAccessMode, diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index c69d1b7ef5..54a7e7c52f 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -770,6 +770,7 @@ typedef enum { VIR_DOMAIN_FS_DRIVER_TYPE_LOOP, VIR_DOMAIN_FS_DRIVER_TYPE_NBD, VIR_DOMAIN_FS_DRIVER_TYPE_PLOOP, + VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIO_FS, VIR_DOMAIN_FS_DRIVER_TYPE_LAST } virDomainFSDriverType; diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index cbe601099f..985fcdd215 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -2697,6 +2697,9 @@ qemuBuildFilesystemCommandLine(virCommandPtr cmd, if (qemuBuildFSDevCommandLine(cmd, def->fss[i], def, qemuCaps) < 0) return -1; break; + case VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIO_FS: + /* TODO: vhost-user-fs-pci */ + return 0; case VIR_DOMAIN_FS_DRIVER_TYPE_LOOP: case VIR_DOMAIN_FS_DRIVER_TYPE_NBD: case VIR_DOMAIN_FS_DRIVER_TYPE_PLOOP: diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 667cc89072..334039005c 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -7110,6 +7110,10 @@ qemuDomainDeviceDefValidateFS(virDomainFSDefPtr fs, _("Filesystem driver type not supported")); return -1; + case VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIO_FS: + /* TODO: vhost-user-fs-pci */ + return -0; + case VIR_DOMAIN_FS_DRIVER_TYPE_LAST: default: virReportEnumRangeError(virDomainFSDriverType, fs->fsdriver); diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c index ea5327d475..482373991c 100644 --- a/src/qemu/qemu_domain_address.c +++ b/src/qemu/qemu_domain_address.c @@ -692,6 +692,9 @@ qemuDomainDeviceCalculatePCIConnectFlags(virDomainDeviceDefPtr dev, case VIR_DOMAIN_FS_MODEL_LAST: break; } + case VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIO_FS: + /* vhost-user-fs-pci */ + return virtioFlags; case VIR_DOMAIN_FS_DRIVER_TYPE_LOOP: case VIR_DOMAIN_FS_DRIVER_TYPE_NBD: diff --git a/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.xml b/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.xml index 284ee329b3..ecd5b33a28 100644 --- a/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.xml +++ b/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.xml @@ -25,6 +25,12 @@ <emulator>/usr/bin/qemu-system-x86_64</emulator> <controller type='usb' index='0' model='none'/> <controller type='pci' index='0' model='pci-root'/> + <filesystem type='mount' accessmode='passthrough'> + <driver type='virtio-fs'/> + <source dir='/path'/> + <target dir='/path'/> + <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/> + </filesystem> <input type='mouse' bus='ps2'/> <input type='keyboard' bus='ps2'/> <memballoon model='none'/> -- 2.21.0

Start virtiofsd for each <filesystem> device using it. Pre-create the socket for communication with QEMU and pass it to virtiofsd. Note that virtiofsd needs to run as root. --- src/conf/domain_conf.h | 1 + src/qemu/qemu_extdevice.c | 191 ++++++++++++++++++++++++++++++++++++++ tests/qemuxml2argvtest.c | 6 ++ 3 files changed, 198 insertions(+) diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 54a7e7c52f..78f88a0c2f 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -817,6 +817,7 @@ struct _virDomainFSDef { unsigned long long space_soft_limit; /* in bytes */ bool symlinksResolved; virDomainVirtioOptionsPtr virtio; + char *vhost_user_fs_path; /* TODO put this in private data */ }; diff --git a/src/qemu/qemu_extdevice.c b/src/qemu/qemu_extdevice.c index 463f76c21a..cb44ca325f 100644 --- a/src/qemu/qemu_extdevice.c +++ b/src/qemu/qemu_extdevice.c @@ -20,6 +20,7 @@ #include <config.h> +#include "qemu_command.h" #include "qemu_extdevice.h" #include "qemu_vhost_user_gpu.h" #include "qemu_domain.h" @@ -149,6 +150,178 @@ qemuExtDevicesCleanupHost(virQEMUDriverPtr driver, qemuExtTPMCleanupHost(def); } +static char * +qemuExtVirtioFSCreatePidFilename(virQEMUDriverConfigPtr cfg, + const virDomainDef *def, + const char *alias) +{ + g_autofree char *shortName = NULL; + g_autofree char *name = NULL; + + if (!(shortName = virDomainDefGetShortName(def)) || + virAsprintf(&name, "%s-%s-virtiofsd", shortName, alias) < 0) + return NULL; + + return virPidFileBuildPath(cfg->stateDir, name); +} + + +static char * +qemuExtVirtioFSCreateSocketFilename(virQEMUDriverConfigPtr cfg, + const virDomainDef *def, + const char *alias) +{ + g_autofree char *shortName = NULL; + g_autofree char *name = NULL; + + if (!(shortName = virDomainDefGetShortName(def)) || + virAsprintf(&name, "%s-%s-virtiofsd", shortName, alias) < 0) + return NULL; + + return virFileBuildPath(cfg->stateDir, name, ".sock"); +} + + +static int +qemuExtVirtioFSdStart(virQEMUDriverPtr driver, + virDomainObjPtr vm, + virDomainFSDefPtr fs) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + g_autoptr(virCommand) cmd = NULL; + g_autofree char *pidfile = NULL; + char errbuf[1024] = { 0 }; + virDomainChrSourceDefPtr chrdev = virDomainChrSourceDefNew(NULL); + pid_t pid = (pid_t) -1; + VIR_AUTOCLOSE errfd = -1; + VIR_AUTOCLOSE fd = -1; + int exitstatus = 0; + int cmdret = 0; + int ret = -1; + int rc; + + chrdev->type = VIR_DOMAIN_CHR_TYPE_UNIX; + chrdev->data.nix.listen = true; + + if (!(pidfile = qemuExtVirtioFSCreatePidFilename(cfg, vm->def, fs->info.alias))) + goto cleanup; + + if (!(chrdev->data.nix.path = qemuExtVirtioFSCreateSocketFilename(cfg, vm->def, fs->info.alias))) + goto cleanup; + + if (qemuSecuritySetDaemonSocketLabel(driver->securityManager, vm->def) < 0) + goto cleanup; + fd = qemuOpenChrChardevUNIXSocket(chrdev); + if (fd < 0) + goto cleanup; + if (qemuSecurityClearSocketLabel(driver->securityManager, vm->def) < 0) + goto cleanup; + + /* TODO: do not call this here */ + virDomainChrDef chr = { .source = chrdev }; + if (qemuSecuritySetChardevLabel(driver, vm, &chr) < 0) + goto cleanup; + + /* TODO: configure path */ + if (!(cmd = virCommandNew("/usr/libexec/virtiofsd"))) + goto cleanup; + + virCommandSetPidFile(cmd, pidfile); + virCommandSetErrorFD(cmd, &errfd); + virCommandDaemonize(cmd); + + virCommandAddArg(cmd, "--syslog"); + virCommandAddArgFormat(cmd, "--fd=%d", fd); + virCommandPassFD(cmd, fd, VIR_COMMAND_PASS_FD_CLOSE_PARENT); + fd = -1; + + virCommandAddArg(cmd, "-o"); + /* TODO: validate path? */ + virCommandAddArgFormat(cmd, "source=%s", fs->src->path); + virCommandAddArg(cmd, "-d"); + + if (qemuExtDeviceLogCommand(driver, vm, cmd, "virtiofsd") < 0) + goto cleanup; + + cmdret = virCommandRun(cmd, &exitstatus); + + if (cmdret < 0 || exitstatus != 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Could not start 'virtiofsd'. exitstatus: %d"), exitstatus); + goto error; + } + + rc = virPidFileReadPath(pidfile, &pid); + if (rc < 0) { + virReportSystemError(-rc, + _("Unable to read virtiofsd pidfile '%s'"), + pidfile); + goto error; + } + + if (virProcessKill(pid, 0) != 0) { + if (saferead(errfd, errbuf, sizeof(errbuf) - 1) < 0) { + virReportSystemError(errno, "%s", + _("virtiofsd died unexpectedly")); + } else { + virReportError(VIR_ERR_OPERATION_FAILED, + _("virtiofsd died and reported: %s"), errbuf); + } + goto error; + } + + fs->vhost_user_fs_path = g_steal_pointer(&chrdev->data.nix.path); + ret = 0; + + cleanup: + if (chrdev->data.nix.path) + unlink(chrdev->data.nix.path); + virObjectUnref(chrdev); + return ret; + + error: + if (pid != -1) + virProcessKillPainfully(pid, true); + if (pidfile) + unlink(pidfile); + goto cleanup; +} + + +static void +qemuExtVirtioFSdStop(virQEMUDriverPtr driver, + virDomainObjPtr vm, + virDomainFSDefPtr fs) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + g_autofree char *pidfile = NULL; + virErrorPtr orig_err; + pid_t pid = -1; + int rc; + + virErrorPreserveLast(&orig_err); + + if (!(pidfile = qemuExtVirtioFSCreatePidFilename(cfg, vm->def, fs->info.alias))) + goto cleanup; + + rc = virPidFileReadPathIfAlive(pidfile, &pid, NULL); + if (rc >= 0 && pid != (pid_t) -1) + virProcessKillPainfully(pid, true); + + if (unlink(pidfile) < 0 && + errno != ENOENT) { + virReportSystemError(errno, + _("Unable to remove stale pidfile %s"), + pidfile); + } + + if (fs->vhost_user_fs_path) + unlink(fs->vhost_user_fs_path); + + cleanup: + virErrorRestore(&orig_err); +} + int qemuExtDevicesStart(virQEMUDriverPtr driver, @@ -184,6 +357,17 @@ qemuExtDevicesStart(virQEMUDriverPtr driver, return -1; } + for (i = 0; i < def->nfss; i++) { + /* FIXME: use private data */ + virDomainFSDefPtr fs = def->fss[i]; + + if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIO_FS) { + if (qemuExtVirtioFSdStart(driver, vm, fs) < 0) + return -1; + } + } + + return ret; } @@ -215,6 +399,13 @@ qemuExtDevicesStop(virQEMUDriverPtr driver, if (slirp) qemuSlirpStop(slirp, vm, driver, net, false); } + + for (i = 0; i < def->nfss; i++) { + virDomainFSDefPtr fs = def->fss[i]; + + if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIO_FS) + qemuExtVirtioFSdStop(driver, vm, fs); + } } diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 24df0a48e7..bf3366f953 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -501,6 +501,12 @@ testCompareXMLToArgv(const void *data) } } + for (i = 0; i < vm->def->nfss; i++) { + virDomainFSDefPtr fs = vm->def->fss[i]; + char *s = g_strdup_printf("/tmp/lib/domain--1-guest/fs%zu.vhost-fs.sock", i); + fs->vhost_user_fs_path = s; + } + if (vm->def->vsock) { virDomainVsockDefPtr vsock = vm->def->vsock; qemuDomainVsockPrivatePtr vsockPriv = -- 2.21.0

On Fri, Nov 01, 2019 at 01:16:06PM +0100, Ján Tomko wrote: IIUC, this is a socket rather than a file, so I'd call it "vhostuser_fs_sock" personally. g_strdup_printf so we can assume this always succeeds likewise we can assume this aborts on oom Same comments No ned to check for failure of these two. Indeed, needs to be in the security manager code. In $QEMU/docs/interop/vhost-user.json there's a specificiation for how we're expected to locate the virtiofsd binary based on metadata files. Same approach we're using for locating firmware files basically. I feel like maybe we should be using a logfile as we do for QEMU, via virtlogd ? IIRC, it was decided intended that we'd have a dbus interface to virtiofsd, one of whose jobs would be a way to turn on / off logging, similar to our own virt-admin facility. Could perhaps be useful to have debug on from startip, but a qemu.conf setting is probably sufficient for this. I feel like we should be putting it into the cgroup for the VM as a psuedo "emulator thread". We'll also need to make sure that virtiofsd gets given an SELinux policy, with sVirt MCS tag associated with it. This raises an interesting question though, because from libvirt's POV we're not supposed to know what binary we're actually invoking. We're supposed to follow the vhost-user.json spec to find an binary, allowing the user to drop in arbitrary impls. Each impl though may have differeing SELinux requirements, so we need to figure out what SElinux domain type to apply. I think this means vhost-user.json needs to specify the desired SELinux domain for each vhost user binary. Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|