On Thu, Jan 30, 2020 at 06:06:20PM +0100, Ján Tomko wrote: ...this shouldn't be here still requires Why are you disabling list-ification here & throughout the file ? I'm not understanding from this how big the files will be for virtiofsd ? How much memory should users expect to be needed per-guest per-virtiofsd ? Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|

On Mon, Feb 03, 2020 at 04:17:31PM +0000, Daniel P. Berrangé wrote: The plan was to leave it in every version posted to the mailing list, for anyone who might find it through the archives. I could not get it to do what I want - it kept resetting to 1. Jano

Add a 'virtiofsd_debug' option for tuning whether to run virtiofsd in debug mode. Signed-off-by: Ján Tomko <jtomko(a)redhat.com&gt; --- src/qemu/libvirtd_qemu.aug | 1 + src/qemu/qemu.conf | 7 +++++++ src/qemu/qemu_conf.c | 2 ++ src/qemu/qemu_conf.h | 1 + src/qemu/test_libvirtd_qemu.aug.in | 1 + 5 files changed, 12 insertions(+) diff --git a/src/qemu/libvirtd_qemu.aug b/src/qemu/libvirtd_qemu.aug index 557b6f38f8..3014fa6b86 100644 --- a/src/qemu/libvirtd_qemu.aug +++ b/src/qemu/libvirtd_qemu.aug @@ -116,6 +116,7 @@ module Libvirtd_qemu = let nvram_entry = str_array_entry "nvram" let debug_level_entry = int_entry "gluster_debug_level" + | bool_entry "virtiofsd_debug" let memory_entry = str_entry "memory_backing_dir" diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf index b6805ffc41..e82c1b5bd5 100644 --- a/src/qemu/qemu.conf +++ b/src/qemu/qemu.conf @@ -809,6 +809,13 @@ # #gluster_debug_level = 9 +# virtiofsd debug +# +# Whether to enable the debugging output of the virtiofsd daemon. +# Possible values are 0 or 1. +# +#virtiofsd_debug = 1 + # To enhance security, QEMU driver is capable of creating private namespaces # for each domain started. Well, so far only "mount" namespace is supported. If # enabled it means qemu process is unable to see all the devices on the system, diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c index e5051027fc..61f694d438 100644 --- a/src/qemu/qemu_conf.c +++ b/src/qemu/qemu_conf.c @@ -837,6 +837,8 @@ virQEMUDriverConfigLoadDebugEntry(virQEMUDriverConfigPtr cfg, { if (virConfGetValueUInt(conf, "gluster_debug_level", &cfg->glusterDebugLevel) < 0) return -1; + if (virConfGetValueBool(conf, "virtiofsd_debug", &cfg->virtiofsdDebug) < 0) + return -1; return 0; } diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index cedf232223..3ce9566b71 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -202,6 +202,7 @@ struct _virQEMUDriverConfig { virFirmwarePtr *firmwares; size_t nfirmwares; unsigned int glusterDebugLevel; + bool virtiofsdDebug; char *memoryBackingDir; diff --git a/src/qemu/test_libvirtd_qemu.aug.in b/src/qemu/test_libvirtd_qemu.aug.in index dd90edf687..fca9a942c9 100644 --- a/src/qemu/test_libvirtd_qemu.aug.in +++ b/src/qemu/test_libvirtd_qemu.aug.in @@ -98,6 +98,7 @@ module Test_libvirtd_qemu = } { "stdio_handler" = "logd" } { "gluster_debug_level" = "9" } +{ "virtiofsd_debug" = "1" } { "namespaces" { "1" = "mount" } } -- 2.21.0

Reject unsupported configurations. Signed-off-by: Ján Tomko <jtomko(a)redhat.com&gt; --- src/qemu/qemu_domain.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 6367f5394e..b5d5812ff8 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -8314,8 +8314,32 @@ qemuDomainDeviceDefValidateFS(virDomainFSDefPtr fs, return -1; case VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS: - /* TODO: vhost-user-fs-pci */ - return 0; + if (fs->accessmode != VIR_DOMAIN_FS_ACCESSMODE_PASSTHROUGH) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("virtiofs only supports passthrough accessmode")); + return -1; + } + if (fs->wrpolicy != VIR_DOMAIN_FS_WRPOLICY_DEFAULT) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("virtiofs does not support wrpolicy")); + return -1; + } + if (fs->model != VIR_DOMAIN_FS_MODEL_DEFAULT) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("virtiofs does not support model")); + return -1; + } + if (fs->format != VIR_STORAGE_FILE_NONE) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("virtiofs does not support format")); + return -1; + } + if (def->mem.access != VIR_DOMAIN_MEMORY_ACCESS_SHARED) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("virtiofs requires shared memory")); + return -1; + } + break; case VIR_DOMAIN_FS_DRIVER_TYPE_LAST: default: -- 2.21.0

On Thu, Jan 30, 2020 at 06:06:24PM +0100, Ján Tomko wrote: This requires the following <memoryBacking> config, right? <memoryBacking> <access mode='shared'/> </memoryBacking> I think the <memoryBacking> isn't mandatory because the memory access policy can be over written by <numa> config like as: <numa> <cell id='0' cpus='0-31' memory='8912896' unit='KiB' memAccess='shared'/> </numa> And, I suppose virtiofsd requires one or more numa nodes. How about adding following function to check the mem config instead of the if statement? diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index 493864b854..b3b5cb2054 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -8203,6 +8203,39 @@ qemuDomainDeviceDefValidateIOMMU(const virDomainIOMMUDef *iommu, return 0; } +static int +memconfIsSufficientForVirtiofs(const virDomainDef *def) +{ + virDomainNumaPtr numa = def->numa; + virDomainMemoryAccess memAccess = def->mem.access; + virDomainMemoryAccess NodememAccess; + size_t nodes; + size_t i; + + nodes = virDomainNumaGetNodeCount(numa); + if (!nodes) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("virtiofs requires one or more numa nodes")); + return 0; + } + + for (i = 0; i < nodes; i++) { + NodememAccess = virDomainNumaGetNodeMemoryAccessMode(numa, i); + if (NodememAccess == VIR_DOMAIN_MEMORY_ACCESS_PRIVATE) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("virtiofs requires shared memory for node[%ld]"), i); + return 0; + } else if ((NodememAccess == VIR_DOMAIN_MEMORY_ACCESS_DEFAULT) && + (memAccess != VIR_DOMAIN_MEMORY_ACCESS_SHARED)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, + _("virtiofs requires shared memory for node[%ld]"), i); + return 0; + } + } + + return 1; +} + static int qemuDomainDeviceDefValidateFS(virDomainFSDefPtr fs, @@ -8256,9 +8289,7 @@ qemuDomainDeviceDefValidateFS(virDomainFSDefPtr fs, _("virtiofs does not support format")); return -1; } - if (def->mem.access != VIR_DOMAIN_MEMORY_ACCESS_SHARED) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("virtiofs requires shared memory")); + if (!memconfIsSufficientForVirtiofs(def)) { return -1; } break; --- Thanks, Masa

Start virtiofsd for each <filesystem> device using it. Pre-create the socket for communication with QEMU and pass it to virtiofsd. Note that virtiofsd needs to run as root. https://bugzilla.redhat.com/show_bug.cgi?id=1694166 Introduced by QEMU commit a43efa34c7d7b628cbf1ec0fe60043e5c91043ea Signed-off-by: Ján Tomko <jtomko(a)redhat.com&gt; --- po/POTFILES.in | 1 + src/qemu/Makefile.inc.am | 2 + src/qemu/qemu_domain.c | 5 +- src/qemu/qemu_domain.h | 2 +- src/qemu/qemu_extdevice.c | 20 ++- src/qemu/qemu_virtiofs.c | 290 ++++++++++++++++++++++++++++++++++++++ src/qemu/qemu_virtiofs.h | 38 +++++ tests/qemuxml2argvtest.c | 11 ++ 8 files changed, 366 insertions(+), 3 deletions(-) create mode 100644 src/qemu/qemu_virtiofs.c create mode 100644 src/qemu/qemu_virtiofs.h diff --git a/po/POTFILES.in b/po/POTFILES.in index c18e21615f..813fb24199 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -168,6 +168,7 @@ @SRCDIR(a)/src/qemu/qemu_tpm.c @SRCDIR(a)/src/qemu/qemu_vhost_user.c @SRCDIR(a)/src/qemu/qemu_vhost_user_gpu.c +@SRCDIR(a)/src/qemu/qemu_virtiofs.c @SRCDIR(a)/src/remote/remote_daemon.c @SRCDIR(a)/src/remote/remote_daemon_config.c @SRCDIR(a)/src/remote/remote_daemon_dispatch.c diff --git a/src/qemu/Makefile.inc.am b/src/qemu/Makefile.inc.am index d04a87e659..7a205b4da6 100644 --- a/src/qemu/Makefile.inc.am +++ b/src/qemu/Makefile.inc.am @@ -67,6 +67,8 @@ QEMU_DRIVER_SOURCES = \ qemu/qemu_vhost_user.h \ qemu/qemu_vhost_user_gpu.c \ qemu/qemu_vhost_user_gpu.h \ + qemu/qemu_virtiofs.c \ + qemu/qemu_virtiofs.h \ qemu/qemu_checkpoint.c \ qemu/qemu_checkpoint.h \ qemu/qemu_backup.c \ diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index b5d5812ff8..3064c33ca8 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -1440,8 +1440,11 @@ qemuDomainFSPrivateNew(void) static void -qemuDomainFSPrivateDispose(void *obj G_GNUC_UNUSED) +qemuDomainFSPrivateDispose(void *obj) { + qemuDomainFSPrivatePtr priv = obj; + + g_free(priv->vhostuser_fs_sock); } static virClassPtr qemuDomainVideoPrivateClass; diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index c581b3a162..83150e4e6d 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -568,7 +568,7 @@ typedef qemuDomainFSPrivate *qemuDomainFSPrivatePtr; struct _qemuDomainFSPrivate { virObject parent; - int dummy; + char *vhostuser_fs_sock; }; diff --git a/src/qemu/qemu_extdevice.c b/src/qemu/qemu_extdevice.c index 7f3bb104d9..5103d4921c 100644 --- a/src/qemu/qemu_extdevice.c +++ b/src/qemu/qemu_extdevice.c @@ -20,11 +20,13 @@ #include <config.h> +#include "qemu_command.h" #include "qemu_extdevice.h" #include "qemu_vhost_user_gpu.h" #include "qemu_domain.h" #include "qemu_tpm.h" #include "qemu_slirp.h" +#include "qemu_virtiofs.h" #include "viralloc.h" #include "virlog.h" @@ -153,7 +155,7 @@ qemuExtDevicesCleanupHost(virQEMUDriverPtr driver, int qemuExtDevicesStart(virQEMUDriverPtr driver, virDomainObjPtr vm, - virLogManagerPtr logManager G_GNUC_UNUSED, + virLogManagerPtr logManager, bool incomingMigration) { virDomainDefPtr def = vm->def; @@ -183,6 +185,15 @@ qemuExtDevicesStart(virQEMUDriverPtr driver, return -1; } + for (i = 0; i < def->nfss; i++) { + virDomainFSDefPtr fs = def->fss[i]; + + if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS) { + if (qemuVirtioFSStart(logManager, driver, vm, fs) < 0) + return -1; + } + } + return 0; } @@ -214,6 +225,13 @@ qemuExtDevicesStop(virQEMUDriverPtr driver, if (slirp) qemuSlirpStop(slirp, vm, driver, net, false); } + + for (i = 0; i < def->nfss; i++) { + virDomainFSDefPtr fs = def->fss[i]; + + if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS) + qemuVirtioFSStop(driver, vm, fs); + } } diff --git a/src/qemu/qemu_virtiofs.c b/src/qemu/qemu_virtiofs.c new file mode 100644 index 0000000000..4aa8eaed2c --- /dev/null +++ b/src/qemu/qemu_virtiofs.c @@ -0,0 +1,290 @@ +/* + * qemu_virtiofs.c: virtiofs support + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + */ + +#include <config.h> + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include "logging/log_manager.h" +#include "virlog.h" +#include "qemu_command.h" +#include "qemu_conf.h" +#include "qemu_extdevice.h" +#include "qemu_security.h" +#include "qemu_virtiofs.h" +#include "virpidfile.h" + +#define VIR_FROM_THIS VIR_FROM_QEMU + + +char * +qemuVirtioFSCreatePidFilename(virQEMUDriverConfigPtr cfg, + const virDomainDef *def, + const char *alias) +{ + g_autofree char *shortName = NULL; + g_autofree char *name = NULL; + + if (!(shortName = virDomainDefGetShortName(def))) + return NULL; + + name = g_strdup_printf("%s-%s-virtiofsd", shortName, alias); + + return virPidFileBuildPath(cfg->stateDir, name); +} + + +char * +qemuVirtioFSCreateSocketFilename(virDomainObjPtr vm, + const char *alias) +{ + qemuDomainObjPrivatePtr priv = vm->privateData; + + return virFileBuildPath(priv->libDir, alias, "-virtiofsd.sock"); +} + + +static char * +qemuVirtioFSCreateLogFilename(virQEMUDriverConfigPtr cfg, + const virDomainDef *def, + const char *alias) +{ + g_autofree char *name = NULL; + + name = g_strdup_printf("%s-%s", def->name, alias); + + return virFileBuildPath(cfg->logDir, name, "-virtiofsd.log"); +} + + +static int +qemuVirtioFSOpenChardev(virQEMUDriverPtr driver, + virDomainObjPtr vm, + const char *socket_path) +{ + virDomainChrSourceDefPtr chrdev = virDomainChrSourceDefNew(NULL); + virDomainChrDef chr = { .source = chrdev }; + VIR_AUTOCLOSE fd = -1; + int ret = -1; + + chrdev->type = VIR_DOMAIN_CHR_TYPE_UNIX; + chrdev->data.nix.listen = true; + chrdev->data.nix.path = g_strdup(socket_path); + + if (qemuSecuritySetDaemonSocketLabel(driver->securityManager, vm->def) < 0) + goto cleanup; + fd = qemuOpenChrChardevUNIXSocket(chrdev); + if (fd < 0) { + ignore_value(qemuSecurityClearSocketLabel(driver->securityManager, vm->def)); + goto cleanup; + } + if (qemuSecurityClearSocketLabel(driver->securityManager, vm->def) < 0) + goto cleanup; + + if (qemuSecuritySetChardevLabel(driver, vm, &chr) < 0) + goto cleanup; + + ret = fd; + fd = -1; + + cleanup: + virObjectUnref(chrdev); + return ret; +} + +static virCommandPtr +qemuVirtioFSBuildCommandLine(virQEMUDriverConfigPtr cfg, + virDomainFSDefPtr fs, + int *fd) +{ + g_autoptr(virCommand) cmd = NULL; + g_auto(virBuffer) opts = VIR_BUFFER_INITIALIZER; + + if (!(cmd = virCommandNew(fs->binary))) + return NULL; + + virCommandAddArgFormat(cmd, "--fd=%d", *fd); + virCommandPassFD(cmd, *fd, VIR_COMMAND_PASS_FD_CLOSE_PARENT); + *fd = -1; + + virCommandAddArg(cmd, "-o"); + virBufferAsprintf(&opts, "source=%s,", fs->src->path); + if (fs->cache) + virBufferAsprintf(&opts, "cache=%s,", virDomainFSCacheModeTypeToString(fs->cache)); + + if (fs->xattr == VIR_TRISTATE_SWITCH_ON) + virBufferAddLit(&opts, "xattr,"); + else if (fs->xattr == VIR_TRISTATE_SWITCH_OFF) + virBufferAddLit(&opts, "no_xattr,"); + + if (fs->flock == VIR_TRISTATE_SWITCH_ON) + virBufferAddLit(&opts, "flock,"); + else if (fs->flock == VIR_TRISTATE_SWITCH_OFF) + virBufferAddLit(&opts, "no_flock,"); + + if (fs->posix_lock == VIR_TRISTATE_SWITCH_ON) + virBufferAddLit(&opts, "posix_lock,"); + else if (fs->posix_lock == VIR_TRISTATE_SWITCH_OFF) + virBufferAddLit(&opts, "no_posix_lock,"); + + virBufferTrim(&opts, ",", -1); + + virCommandAddArgBuffer(cmd, &opts); + if (cfg->virtiofsdDebug) + virCommandAddArg(cmd, "-d"); + + return g_steal_pointer(&cmd); +} + +int +qemuVirtioFSStart(virLogManagerPtr logManager, + virQEMUDriverPtr driver, + virDomainObjPtr vm, + virDomainFSDefPtr fs) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + g_autoptr(virCommand) cmd = NULL; + g_autofree char *socket_path = NULL; + g_autofree char *pidfile = NULL; + g_autofree char *logpath = NULL; + pid_t pid = (pid_t) -1; + VIR_AUTOCLOSE fd = -1; + VIR_AUTOCLOSE logfd = -1; + int ret = -1; + int rc; + + if (!(pidfile = qemuVirtioFSCreatePidFilename(cfg, vm->def, fs->info.alias))) + goto cleanup; + + if (!(socket_path = qemuVirtioFSCreateSocketFilename(vm, fs->info.alias))) + goto cleanup; + + if ((fd = qemuVirtioFSOpenChardev(driver, vm, socket_path)) < 0) + goto cleanup; + + logpath = qemuVirtioFSCreateLogFilename(cfg, vm->def, fs->info.alias); + + if (cfg->stdioLogD) { + if ((logfd = virLogManagerDomainOpenLogFile(logManager, + "qemu", + vm->def->uuid, + vm->def->name, + logpath, + 0, + NULL, NULL)) < 0) + goto cleanup; + } else { + if ((logfd = open(logpath, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR)) < 0) { + virReportSystemError(errno, _("failed to create logfile %s"), + logpath); + goto cleanup; + } + if (virSetCloseExec(logfd) < 0) { + virReportSystemError(errno, _("failed to set close-on-exec flag on %s"), + logpath); + goto error; + } + } + + if (!(cmd = qemuVirtioFSBuildCommandLine(cfg, fs, &fd))) + goto cleanup; + + virCommandSetPidFile(cmd, pidfile); + virCommandSetOutputFD(cmd, &logfd); + virCommandSetErrorFD(cmd, &logfd); + virCommandNonblockingFDs(cmd); + virCommandDaemonize(cmd); + + if (qemuExtDeviceLogCommand(driver, vm, cmd, "virtiofsd") < 0) + goto cleanup; + + rc = virCommandRun(cmd, NULL); + logfd = -1; + + if (rc < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Could not start 'virtiofsd'")); + goto error; + } + + rc = virPidFileReadPath(pidfile, &pid); + if (rc < 0) { + virReportSystemError(-rc, + _("Unable to read virtiofsd pidfile '%s'"), + pidfile); + goto error; + } + + if (virProcessKill(pid, 0) != 0) { + virReportSystemError(errno, "%s", + _("virtiofsd died unexpectedly")); + goto error; + } + + QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock = g_steal_pointer(&socket_path); + ret = 0; + + cleanup: + if (socket_path) + unlink(socket_path); + return ret; + + error: + if (pid != -1) + virProcessKillPainfully(pid, true); + if (pidfile) + unlink(pidfile); + goto cleanup; +} + + +void +qemuVirtioFSStop(virQEMUDriverPtr driver, + virDomainObjPtr vm, + virDomainFSDefPtr fs) +{ + g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver); + g_autofree char *pidfile = NULL; + virErrorPtr orig_err; + pid_t pid = -1; + int rc; + + virErrorPreserveLast(&orig_err); + + if (!(pidfile = qemuVirtioFSCreatePidFilename(cfg, vm->def, fs->info.alias))) + goto cleanup; + + rc = virPidFileReadPathIfAlive(pidfile, &pid, NULL); + if (rc >= 0 && pid != (pid_t) -1) + virProcessKillPainfully(pid, true); + + if (unlink(pidfile) < 0 && + errno != ENOENT) { + virReportSystemError(errno, + _("Unable to remove stale pidfile %s"), + pidfile); + } + + if (QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock) + unlink(QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock); + + cleanup: + virErrorRestore(&orig_err); +} diff --git a/src/qemu/qemu_virtiofs.h b/src/qemu/qemu_virtiofs.h new file mode 100644 index 0000000000..49db807b19 --- /dev/null +++ b/src/qemu/qemu_virtiofs.h @@ -0,0 +1,38 @@ +/* + * qemu_virtiofs.h: virtiofs support + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see + * <http://www.gnu.org/licenses/>;. + */ + +#pragma once + + +char * +qemuVirtioFSCreatePidFilename(virQEMUDriverConfigPtr cfg, + const virDomainDef *def, + const char *alias); +char * +qemuVirtioFSCreateSocketFilename(virDomainObjPtr vm, + const char *alias); + +int +qemuVirtioFSStart(virLogManagerPtr logManager, + virQEMUDriverPtr driver, + virDomainObjPtr vm, + virDomainFSDefPtr fs); +void +qemuVirtioFSStop(virQEMUDriverPtr driver, + virDomainObjPtr vm, + virDomainFSDefPtr fs); diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index a36183bf34..f268d336a6 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -496,6 +496,17 @@ testCompareXMLToArgv(const void *data) } } + for (i = 0; i < vm->def->nfss; i++) { + virDomainFSDefPtr fs = vm->def->fss[i]; + char *s; + + if (fs->fsdriver != VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS) + continue; + + s = g_strdup_printf("/tmp/lib/domain--1-guest/fs%zu.vhost-fs.sock", i); + QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock = s; + } + if (vm->def->vsock) { virDomainVsockDefPtr vsock = vm->def->vsock; qemuDomainVsockPrivatePtr vsockPriv = -- 2.21.0

On Thu, Jan 30, 2020 at 06:06:26PM +0100, Ján Tomko wrote: How about adding close the logfile fd here? That is because virtlogd keeps opening the fd even after the guest is shutdowned. # virsh shutdown guest Domain guest is being shutdown # virsh list Id Name State -------------------- # virsh start guest error: Failed to start domain guest error: Cannot open log file: '/var/log/libvirt/qemu/guest-fs0-virtiofsd.log': Device or resource busy # lsof | grep /var/log/libvirt/qemu/guest-fs0-virtiofsd.log virtlogd 5133 root 19w REG 253,0 0 35189059 /var/log/libvirt/qemu/guest-fs0-virtiofsd.log # Sample patch to close logfile fd: diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c index b3b5cb2054..7d5b7775fa 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c @@ -1439,6 +1439,7 @@ qemuDomainFSPrivateDispose(void *obj) qemuDomainFSPrivatePtr priv = obj; g_free(priv->vhostuser_fs_sock); + VIR_FORCE_CLOSE(priv->logfd); } static virClassPtr qemuDomainVideoPrivateClass; diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h index 83150e4e6d..e4b3ac471f 100644 --- a/src/qemu/qemu_domain.h +++ b/src/qemu/qemu_domain.h @@ -569,6 +569,7 @@ struct _qemuDomainFSPrivate { virObject parent; char *vhostuser_fs_sock; + int logfd; }; diff --git a/src/qemu/qemu_virtiofs.c b/src/qemu/qemu_virtiofs.c index 4ea8f23fd5..17532bac20 100644 --- a/src/qemu/qemu_virtiofs.c +++ b/src/qemu/qemu_virtiofs.c @@ -217,7 +217,7 @@ qemuVirtioFSStart(virLogManagerPtr logManager, goto cleanup; rc = virCommandRun(cmd, NULL); - logfd = -1; + QEMU_DOMAIN_FS_PRIVATE(fs)->logfd = logfd; if (rc < 0) { virReportError(VIR_ERR_INTERNAL_ERROR, "%s", -- Thanks, Masa

On Tue, Feb 04, 2020 at 05:36:23PM +0100, Ján Tomko wrote: Great, it works! - Masa

Format the 'vhost-user-fs' device on the QEMU command line. This device provides shared file system access using the FUSE protocol carried over virtio. The actual file server is implemented in an external vhost-user-fs device backend process. https://bugzilla.redhat.com/show_bug.cgi?id=1694166 Signed-off-by: Ján Tomko <jtomko(a)redhat.com&gt; --- src/qemu/qemu_command.c | 45 +++++++++++++++++- ...vhost-user-fs-fd-memory.x86_64-latest.args | 39 +++++++++++++++ ...vhost-user-fs-hugepages.x86_64-latest.args | 47 +++++++++++++++++++ tests/qemuxml2argvtest.c | 3 ++ 4 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 tests/qemuxml2argvdata/vhost-user-fs-fd-memory.x86_64-latest.args create mode 100644 tests/qemuxml2argvdata/vhost-user-fs-hugepages.x86_64-latest.args diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 0d7cd42912..8914b4c5be 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -2566,6 +2566,45 @@ qemuBuildDisksCommandLine(virCommandPtr cmd, } +static int +qemuBuildVHostUserFsCommandLine(virCommandPtr cmd, + virDomainFSDef *fs, + const virDomainDef *def, + qemuDomainObjPrivatePtr priv) +{ + g_autofree char *chardev_alias = NULL; + g_auto(virBuffer) opt = VIR_BUFFER_INITIALIZER; + + chardev_alias = g_strdup_printf("chr-vu-%s", fs->info.alias); + + virCommandAddArg(cmd, "-chardev"); + virBufferAddLit(&opt, "socket"); + virBufferAsprintf(&opt, ",id=%s", chardev_alias); + virBufferEscapeString(&opt, ",path=%s", QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock); + virCommandAddArg(cmd, virBufferContentAndReset(&opt)); + + virCommandAddArg(cmd, "-device"); + + if (qemuBuildVirtioDevStr(&opt, "vhost-user-fs", priv->qemuCaps, + VIR_DOMAIN_DEVICE_FS, fs) < 0) + return -1; + + virBufferAsprintf(&opt, ",chardev=%s", chardev_alias); + if (fs->queue_size) + virBufferAsprintf(&opt, ",queue-size=%llu", fs->queue_size); + virBufferAddLit(&opt, ",tag="); + virQEMUBuildBufferEscapeComma(&opt, fs->dst); + if (qemuBuildVirtioOptionsStr(&opt, fs->virtio, priv->qemuCaps) < 0) + return -1; + + if (qemuBuildDeviceAddressStr(&opt, def, &fs->info, priv->qemuCaps) < 0) + return -1; + + virCommandAddArg(cmd, virBufferContentAndReset(&opt)); + return 0; +} + + static char * qemuBuildFSStr(virDomainFSDefPtr fs) { @@ -2658,7 +2697,7 @@ static int qemuBuildFilesystemCommandLine(virCommandPtr cmd, const virDomainDef *def, virQEMUCapsPtr qemuCaps, - qemuDomainObjPrivatePtr priv G_GNUC_UNUSED) + qemuDomainObjPrivatePtr priv) { size_t i; @@ -2673,7 +2712,9 @@ qemuBuildFilesystemCommandLine(virCommandPtr cmd, break; case VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS: - /* TODO: vhost-user-fs-pci */ + /* vhost-user-fs-pci */ + if (qemuBuildVHostUserFsCommandLine(cmd, def->fss[i], def, priv) < 0) + return -1; break; case VIR_DOMAIN_FS_DRIVER_TYPE_LOOP: diff --git a/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.x86_64-latest.args b/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.x86_64-latest.args new file mode 100644 index 0000000000..a7df45a7f0 --- /dev/null +++ b/tests/qemuxml2argvdata/vhost-user-fs-fd-memory.x86_64-latest.args @@ -0,0 +1,39 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/tmp/lib/domain--1-guest \ +USER=test \ +LOGNAME=test \ +XDG_DATA_HOME=/tmp/lib/domain--1-guest/.local/share \ +XDG_CACHE_HOME=/tmp/lib/domain--1-guest/.cache \ +XDG_CONFIG_HOME=/tmp/lib/domain--1-guest/.config \ +QEMU_AUDIO_DRV=none \ +/usr/bin/qemu-system-x86_64 \ +-name guest=guest,debug-threads=on \ +-S \ +-object secret,id=masterKey0,format=raw,\ +file=/tmp/lib/domain--1-guest/master-key.aes \ +-machine pc,accel=kvm,usb=off,dump-guest-core=off \ +-cpu qemu64 \ +-m 14336 \ +-overcommit mem-lock=off \ +-smp 2,sockets=2,cores=1,threads=1 \ +-object memory-backend-file,id=ram-node0,\ +mem-path=/var/lib/libvirt/qemu/ram/libvirt/qemu/-1-guest/ram-node0,share=yes,\ +size=15032385536 \ +-numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \ +-uuid 126f2720-6f8e-45ab-a886-ec9277079a67 \ +-display none \ +-no-user-config \ +-nodefaults \ +-chardev socket,id=charmonitor,fd=1729,server,nowait \ +-mon chardev=charmonitor,id=monitor,mode=control \ +-rtc base=utc \ +-no-shutdown \ +-no-acpi \ +-boot strict=on \ +-chardev socket,id=chr-vu-fs0,path=/tmp/lib/domain--1-guest/fs0.vhost-fs.sock \ +-device vhost-user-fs-pci,chardev=chr-vu-fs0,queue-size=1024,tag=mount_tag,\ +bus=pci.0,addr=0x2 \ +-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,\ +resourcecontrol=deny \ +-msg timestamp=on diff --git a/tests/qemuxml2argvdata/vhost-user-fs-hugepages.x86_64-latest.args b/tests/qemuxml2argvdata/vhost-user-fs-hugepages.x86_64-latest.args new file mode 100644 index 0000000000..39190b8d3e --- /dev/null +++ b/tests/qemuxml2argvdata/vhost-user-fs-hugepages.x86_64-latest.args @@ -0,0 +1,47 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/tmp/lib/domain--1-guest \ +USER=test \ +LOGNAME=test \ +XDG_DATA_HOME=/tmp/lib/domain--1-guest/.local/share \ +XDG_CACHE_HOME=/tmp/lib/domain--1-guest/.cache \ +XDG_CONFIG_HOME=/tmp/lib/domain--1-guest/.config \ +QEMU_AUDIO_DRV=none \ +/usr/bin/qemu-system-x86_64 \ +-name guest=guest,debug-threads=on \ +-S \ +-object secret,id=masterKey0,format=raw,\ +file=/tmp/lib/domain--1-guest/master-key.aes \ +-machine q35,accel=tcg,usb=off,dump-guest-core=off \ +-cpu qemu64 \ +-m 2048 \ +-overcommit mem-lock=off \ +-smp 2,sockets=2,cores=1,threads=1 \ +-object memory-backend-file,id=ram-node0,prealloc=yes,\ +mem-path=/dev/hugepages2M/libvirt/qemu/-1-guest,share=yes,size=2147483648 \ +-numa node,nodeid=0,cpus=0-1,memdev=ram-node0 \ +-uuid 1ccfd97d-5eb4-478a-bbe6-88d254c16db7 \ +-display none \ +-no-user-config \ +-nodefaults \ +-chardev socket,id=charmonitor,fd=1729,server,nowait \ +-mon chardev=charmonitor,id=monitor,mode=control \ +-rtc base=utc \ +-no-shutdown \ +-boot strict=on \ +-device pcie-root-port,port=0x8,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,\ +addr=0x1 \ +-device pcie-root-port,port=0x9,chassis=2,id=pci.2,bus=pcie.0,addr=0x1.0x1 \ +-device pcie-root-port,port=0xa,chassis=3,id=pci.3,bus=pcie.0,addr=0x1.0x2 \ +-device pcie-root-port,port=0xb,chassis=4,id=pci.4,bus=pcie.0,addr=0x1.0x3 \ +-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/guest.qcow2",\ +"node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \ +-blockdev '{"node-name":"libvirt-1-format","read-only":false,"driver":"qcow2",\ +"file":"libvirt-1-storage"}' \ +-device virtio-blk-pci,scsi=off,bus=pci.4,addr=0x0,drive=libvirt-1-format,\ +id=virtio-disk0,bootindex=1 \ +-chardev socket,id=chr-vu-fs0,path=/tmp/lib/domain--1-guest/fs0.vhost-fs.sock \ +-device vhost-user-fs-pci,chardev=chr-vu-fs0,tag=mount_tag,bus=pci.1,addr=0x0 \ +-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,\ +resourcecontrol=deny \ +-msg timestamp=on diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index f268d336a6..5e65223a81 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -3053,6 +3053,9 @@ mymain(void) DO_TEST_CAPS_VER("launch-security-sev", "2.12.0"); + DO_TEST_CAPS_LATEST("vhost-user-fs-fd-memory"); + DO_TEST_CAPS_LATEST("vhost-user-fs-hugepages"); + DO_TEST("riscv64-virt", QEMU_CAPS_DEVICE_VIRTIO_MMIO); DO_TEST("riscv64-virt-pci", -- 2.21.0